§
    ·¯¸j,  ã                   ó  — d Z ddlmZmZ ddlZddlmZ ddlmZ 	 ddlmZm	Z	 dZ
n# e$ r dZ
Y nw xY wd	d
lmZ ddgZ G d„ de¦  «        Z	 	 dddœdee         dee         deee                  dedee         dedededededededee         fd„Zdee         dee         deee                  dededededededededee         fd„Zdee         dee         deee                  dededededededededee         fd„ZdS ) a   SGD with decoupled weight-decay.

References for added functionality:
    Cautious Optimizers: https://arxiv.org/abs/2411.16085
    Why Gradients Rapidly Increase Near the End of Training: https://arxiv.org/abs/2506.02285

Hacked together by Ross Wightman
é    )ÚListÚOptionalN)ÚTensor)Ú	Optimizer)Ú_use_grad_for_differentiableÚ_default_to_fused_or_foreachTFé   )ÚParamsTÚSGDWÚsgdwc                   ó´   ‡ — e Zd Z	 	 	 	 	 dddddddœdededed	ed
edededededee         defˆ fd„Zˆ fd„Zd„ Z	 e
j        ¦   «         dd„¦   «         Zˆ xZS )r   çü©ñÒMbP?ç        FN)ÚcautionÚcorrected_weight_decayÚmaximizeÚforeachÚdifferentiableÚparamsÚlrÚmomentumÚ	dampeningÚweight_decayÚnesterovr   r   r   r   r   c                óH  •— |dk     rt          d|› �¦  «        ‚|dk     rt          d|› �¦  «        ‚|dk     rt          d|› �¦  «        ‚t          ||||||||	|
|¬¦
  «
        }|r|dk    s|dk    rt          d¦  «        ‚t          ¦   «                              ||¦  «         d S )Nr   zInvalid learning rate: zInvalid momentum value: zInvalid weight_decay value: )
r   r   r   r   r   r   r   r   r   r   r   z8Nesterov momentum requires a momentum and zero dampening)Ú
ValueErrorÚdictÚsuperÚ__init__)Úselfr   r   r   r   r   r   r   r   r   r   r   ÚdefaultsÚ	__class__s                €ú_/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/optim/sgdw.pyr   zSGDW.__init__   sã   ø€ ð �Š8ˆ8ÝÐ;°rÐ;Ð;Ñ<Ô<Ð<Ø�cŠ>ˆ>ÝÐB¸ÐBÐBÑCÔCÐCØ˜#ÒÐÝÐJ¸LÐJÐJÑKÔKÐKåØØØØ%ØØØ#9ØØØ)ð
ñ 
ô 
ˆð ð 	Y˜ Qš˜¨)°qª.¨.ÝÐWÑXÔXÐXÝ‰Œ×Ò˜ Ñ*Ô*Ð*Ð*Ð*ó    c                 óf  •— t          ¦   «                              |¦  «         | j        D ]†}|                     dd¦  «         |                     dd¦  «         |                     dd¦  «         |                     dd¦  «         |                     dd ¦  «         |                     dd¦  «         Œ‡d S )Nr   Fr   r   r   r   r   )r   Ú__setstate__Úparam_groupsÚ
setdefault)r    ÚstateÚgroupr"   s      €r#   r&   zSGDW.__setstate__@   sÀ   ø€ Ý‰Œ×Ò˜UÑ#Ô#Ð#ØÔ&ð 	6ð 	6ˆEØ×Ò˜Y¨Ñ.Ô.Ð.Ø×ÒÐ5°uÑ=Ô=Ð=Ø×Ò˜Z¨Ñ/Ô/Ð/Ø×Ò˜Z¨Ñ/Ô/Ð/Ø×Ò˜Y¨Ñ-Ô-Ð-Ø×ÒÐ-¨uÑ5Ô5Ð5Ð5ð	6ð 	6r$   c                 ó,  — d}|d         D ]ˆ}|j         �|                     |¦  «         |                     |j         ¦  «         |j         j        rd}| j        |         }d|vr|                     d ¦  «         Œm|                     |d         ¦  «         Œ‰|S )NFr   TÚmomentum_buffer)ÚgradÚappendÚ	is_sparser)   )r    r*   Úparams_with_gradÚgradsÚmomentum_buffer_listÚhas_sparse_gradÚpr)   s           r#   Ú_init_groupzSGDW._init_groupJ   s¬   € Øˆà�x”ð 	Jð 	JˆAØŒvÐ!Ø ×'Ò'¨Ñ*Ô*Ð*Ø—’˜QœVÑ$Ô$Ð$Ø”6Ô#ð +Ø&*�Oàœ
 1œ�Ø$¨EÐ1Ð1Ø(×/Ò/°Ñ5Ô5Ð5Ð5à(×/Ò/°Ð6GÔ0HÑIÔIÐIøàÐr$   c                 óÚ  — d}|�5t          j        ¦   «         5   |¦   «         }ddd¦  «         n# 1 swxY w Y   | j        D ]©}g }g }g }|                      ||||¦  «        }t	          ||||d         |d         |d         |d         |d         |d         |d         ||d	         |d
         r| j        d         nd¬¦  «         t          ||¦  «        D ]\  }}	| j        |         }
|	|
d<   ŒŒª|S )z±Performs a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r   r   r   r   r   r   r   r   )
r   r   r   r   r   r   r   r3   r   Úmax_lrr,   )ÚtorchÚenable_gradr'   r5   r   r!   Úzipr)   )r    ÚclosureÚlossr*   r0   r1   r2   r3   r4   r,   r)   s              r#   Ústepz	SGDW.step^   s‚  € ð ˆØÐÝÔ"Ñ$Ô$ð !ð !Ø�w‘y”y�ð!ð !ð !ñ !ô !ð !ð !ð !ð !ð !ð !øøøð !ð !ð !ð !ð Ô&ð 	;ð 	;ˆEØ!ÐØˆEØ#%Ð à"×.Ò.¨uÐ6FÈÐOcÑdÔdˆOåØ ØØ$Ø" >Ô2Ø˜zÔ*Ø˜”;Ø Ô,Ø˜zÔ*Ø˜iÔ(Ø˜zÔ*Ø /Ø˜iÔ(Ø.3Ð4LÔ.MÐW�t”} TÔ*Ð*ÐSWðñ ô ð õ" '*Ð*:Ð<PÑ&QÔ&Qð ;ð ;Ñ"��?Øœ
 1œ�Ø+:�Ð'Ñ(Ð(ð;ð ˆs   ˜/¯3¶3)r   r   r   r   F©N)Ú__name__Ú
__module__Ú__qualname__r
   ÚfloatÚboolr   r   r&   r5   r8   Úno_gradr=   Ú__classcell__)r"   s   @r#   r   r      s/  ø€ € € € € ð Ø Ø!Ø"$Ø"ð$+ð "Ø+0Ø"Ø&*Ø#(ð$+ð $+ð $+àð$+ð ð$+ð ð	$+ð
 ð$+ð  ð$+ð ð$+ð ð$+ð %)ð$+ð ð$+ð ˜d”^ð$+ð !ð$+ð $+ð $+ð $+ð $+ð $+ðL6ð 6ð 6ð 6ð 6ðð ð ð( €U„]�_„_ð(ð (ð (ñ „_ð(ð (ð (ð (ð (r$   )r7   r   r1   r2   r3   r   r   r   r   r   r   r   r   r7   c                óž  — t           r}t          t          d¦  «        rh|€6t          j                             ¦   «         st          | dd¬¦  «        \  }}nd}|r-t          j                             ¦   «         rt          d¦  «        ‚nd}|r&t          j                             ¦   «         st          }nt          } || |||||||	|
|||¬¦  «         dS )zlFunctional API that performs SGD algorithm computation.

    See :class:`~torch.optim.SGD` for details.
    Ú"_group_tensors_by_device_and_dtypeNF)r   Ú	use_fusedz6torch.jit.script not supported with foreach optimizers)	r   r   r   r   r   r   r3   r   r7   )
Úhas_recent_ptÚhasattrr   r8   ÚjitÚis_scriptingr   ÚRuntimeErrorÚ_multi_tensor_sgdwÚ_single_tensor_sgdw)r   r1   r2   r3   r   r   r   r   r   r   r   r   r7   Ú_Úfuncs                  r#   r   r   Š   s   € õ, ð �¥Ð,PÑQÔQð Øˆ?õ ”9×)Ò)Ñ+Ô+ð  Ý9¸&ÐQVÐbgÐhÑhÔh‘
��7�7à�àð 	Y•u”y×-Ò-Ñ/Ô/ð 	YÝÐWÑXÔXÐXøàˆàð #•u”y×-Ò-Ñ/Ô/ð #Ý!ˆˆå"ˆà€DØØØØ!ØØØØØØ'ØØðñ ô ð ð ð r$   c       	         óÜ  — t          | ¦  «        D �]Z\  }}|	s||         n||          }|€|n|dz  |z  }|                     d||z  z
  ¦  «         |dk    rü||         }|€,t          j        |¦  «                             ¦   «         }|||<   n-|                     |¦  «                             |d|z
  ¬¦  «         |r{|r|                     ||¬¦  «        }||z  dk                         |j        ¦  «        }| 	                    | 
                    ¦   «                              d¬¦  «        ¦  «         ||z  }n|r|                     ||¬¦  «        }n|}|                     || ¬¦  «         �Œ\d S )Né   ç      ð?r   r	   ©Úalphar   )Úmin)Ú	enumerateÚmul_r8   ÚcloneÚdetachÚadd_ÚaddÚtoÚdtypeÚdiv_ÚmeanÚclamp_)r   r1   r2   r   r   r   r   r   r   r   r3   r7   ÚiÚparamr-   Úwd_scaleÚbufÚmasks                     r#   rO   rO   Ã   s”  € õ ˜fÑ%Ô%ð $ñ $‰ˆˆ5Ø'Ð6ˆu�QŒxˆx¨e°A¬h¨Yˆà˜�2�2¨R°1©W°vÑ-=ˆØ�
Š
�2˜ <Ñ/Ñ/Ñ0Ô0Ð0à�qŠ=ˆ=Ø& qÔ)ˆCàˆ{Ý”k $Ñ'Ô'×.Ò.Ñ0Ô0�Ø*-Ð$ QÑ'Ð'à—’˜Ñ"Ô"×'Ò'¨°A¸	±MÐ'ÑBÔBÐBàð Øð 8ØŸ(š( 3¨h˜(Ñ7Ô7�Cà˜d™
 Qš×*Ò*¨4¬:Ñ6Ô6�Ø—	’	˜$Ÿ)š)™+œ+×,Ò,°Ð,Ñ6Ô6Ñ7Ô7Ð7Ø˜T‘z��àð ØŸ8š8 C¨x˜8Ñ8Ô8�D�Dà�Dà�
Š
�4 ˜sˆ
Ñ#Ô#Ð#Ñ#ð9$ð $r$   c       	         óæ  — t          | ¦  «        dk    rd S t          j        | ||gd¬¦  «        }|                     ¦   «         D �]¬\  \  }}}}|
ot	          d„ |D ¦   «         ¦  «        }|	rt          j        |¦  «        }|€|n|dz  |z  }t          j        |d||z  z
  ¦  «         |dk    �räg }d}t          t          |¦  «        ¦  «        D ])}||         €d} n| 	                    ||         ¦  «         Œ*|r0t          j        ||¦  «         t          j
        ||d|z
  ¬	¦  «         n¶g }t          t          |¦  «        ¦  «        D ]—}||         €=t          j        ||         ¦  «                             ¦   «         x}x||<   |||         <   n;||         }|                     |¦  «                             ||         d|z
  ¬	¦  «         | 	                    |¦  «         Œ˜|r”|rt          j        |||¬	¦  «        }t          j        ||¦  «        }d
„ t#          ||¦  «        D ¦   «         }d„ |D ¦   «         }t          j        |d¦  «         t          j        ||¦  «         t          j        ||¦  «        }n|rt          j
        |||¬	¦  «         n|}|st          j
        ||| ¬	¦  «         �Œit          t          |¦  «        ¦  «        D ]&}||                              ||         | ¬	¦  «         Œ'�Œ®d S )Nr   T)Úwith_indicesc              3   ó$   K  — | ]}|j         V — Œd S r>   )r/   )Ú.0r-   s     r#   ú	<genexpr>z%_multi_tensor_sgdw.<locals>.<genexpr>  s$   è è € Ð8aÐ8aÈD¸¼Ð8aÐ8aÐ8aÐ8aÐ8aÐ8ar$   rS   rT   Fr	   rU   c                 óP   — g | ]#\  }}|d k                          |j        ¦  «        ‘Œ$S )r   )r^   r_   )rk   ÚmÚgs      r#   ú
<listcomp>z&_multi_tensor_sgdw.<locals>.<listcomp>.  s.   € ÐRÐRÐR±°°A˜!˜aš%Ÿš A¤GÑ,Ô,ÐRÐRÐRr$   c                 ó6   — g | ]}|                      ¦   «         ‘ŒS © )ra   )rk   rn   s     r#   rp   z&_multi_tensor_sgdw.<locals>.<listcomp>/  s    € Ð6Ð6Ð6¨1˜aŸfšf™hœhÐ6Ð6Ð6r$   r   )Úlenr   rG   ÚvaluesÚanyr8   Ú_foreach_negÚ_foreach_mul_Úranger.   Ú_foreach_add_rZ   r[   rY   r\   Ú_foreach_addÚ_foreach_mulr:   Ú_foreach_maximum_Ú_foreach_div_)r   r1   r2   r   r   r   r   r   r   r   r3   r7   Úgrouped_tensorsÚdevice_paramsÚdevice_gradsÚdevice_momentum_buffer_listÚindicesÚdevice_has_sparse_gradre   ÚbufsÚall_states_with_momentum_bufferrc   rf   ÚmasksÚ
mask_scales                            r#   rN   rN   ñ   sŒ  € õ ˆ6�{„{�aÒÐØˆåÔBØ	�Ð,Ð-¸DðBñ Bô B€OàQ`×QgÒQgÑQiÔQið 9Bñ 9BÑMÑ	Cˆ-˜Ð'BÀWØ!0Ð!aµSÐ8aÐ8aÐT`Ð8aÑ8aÔ8aÑ5aÔ5aÐàð 	<Ý Ô-¨lÑ;Ô;ˆLà˜�2�2¨R°1©W°vÑ-=ˆÝÔ˜M¨2°¸<Ñ0GÑ+GÑHÔHÐHà�qŠ=‰=ØˆDà.2Ð+Ý�3Ð:Ñ;Ô;Ñ<Ô<ð @ð @�Ø.¨qÔ1Ð9Ø6;Ð3Ø�Eà—K’KÐ ;¸AÔ >Ñ?Ô?Ð?Ð?à.ð %ÝÔ# D¨(Ñ3Ô3Ð3ÝÔ# D¨,¸aÀ)¹mÐLÑLÔLÐLÐLà�Ý�sÐ#>Ñ?Ô?Ñ@Ô@ð %ð %�AØ2°1Ô5Ð=å!œK¨°Q¬Ñ8Ô8×?Ò?ÑAÔAðB˜ð BÐ9¸!Ñ<Ð?SÐT[Ð\]ÔT^Ñ?_Ð?_ð :¸!Ô<˜ØŸš Ñ*Ô*×/Ò/°¸Q´ÀqÈ9Á}Ð/ÑUÔUÐUà—K’K Ñ$Ô$Ð$Ð$àð (Øð Rå Ô-¨l¸DÈÐQÑQÔQ�DåÔ*¨4°Ñ>Ô>�ØRÐR½¸UÀLÑ9QÔ9QÐRÑRÔR�Ø6Ð6°Ð6Ñ6Ô6�
ÝÔ'¨
°DÑ9Ô9Ð9ÝÔ# E¨:Ñ6Ô6Ð6Ý$Ô1°$¸Ñ>Ô>��àð (ÝÔ'¨°dÀ(ÐKÑKÔKÐKÐKà#'�Là%ð 	BÝÔ ¨|ÀBÀ3ÐGÑGÔGÐGÑGõ �3˜}Ñ-Ô-Ñ.Ô.ð Bð B�Ø˜aÔ ×%Ò% l°1¤o¸b¸SÐ%ÑAÔAÐAÐAñBðq9Bð 9Br$   )NN)Ú__doc__Útypingr   r   r8   r   Útorch.optim.optimizerr   r   r   rI   ÚImportErrorÚ_typesr
   Ú__all__r   rC   rB   r   rO   rN   rr   r$   r#   ú<module>rŽ      s#  ððð ð "Ð !Ð !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð Ø +Ð +Ð +Ð +Ð +Ð +ðØ`Ð`Ð`Ð`Ð`Ð`Ð`Ð`Ø€M€MøØð ð ð Ø€M€M€Mðøøøð Ð Ð Ð Ð Ð à�6Ð
€ðnð nð nð nð nˆ9ñ nô nð nðn !%Ø"&ð6ð  #'ð!6ð 6ð 6Ø�V”ð6à�FŒ|ð6ð # 8¨FÔ#3Ô4ð6ð ð6ð ˜$”ð6ð ð6ð ð6ð ð6ð ð6ð ð6ð ð6ð ð6ð  ˜”ð!6ð 6ð 6ð 6ðr+$Ø�V”ð+$à�FŒ|ð+$ð # 8¨FÔ#3Ô4ð+$ð
 ð+$ð ð+$ð ð+$ð ð+$ð ð+$ð ð+$ð ð+$ð ð+$ð ˜”ð+$ð +$ð +$ð +$ð\MBØ�V”ðMBà�FŒ|ðMBð # 8¨FÔ#3Ô4ðMBð
 ðMBð ðMBð ðMBð ðMBð ðMBð ðMBð ðMBð ðMBð ˜”ðMBð MBð MBð MBð MBð MBs   œ
' §1°1