§
    ·¯¸jí-  ã            "       ó¦  — d Z ddlZddlmZmZmZ ddlZddlmZ ddlm	Z	  G d„ de
¦  «        Z G d„ d	e	¦  «        Zd
ee         dee         dee         dee         dee         dee         dededededededededededef"d„Zd
ee         dee         dee         dee         dee         dee         dededededededededededef"d„ZdS )zë Adan Optimizer

Adan: Adaptive Nesterov Momentum Algorithm for Faster Optimizing Deep Models[J]. arXiv preprint arXiv:2208.06677, 2022.
    https://arxiv.org/abs/2208.06677

Implementation adapted from https://github.com/sail-sg/Adan
é    N)ÚListÚOptionalÚTuple)ÚTensor)Ú	Optimizerc                   ó"   — e Zd ZdZdZd„ Zd„ ZdS )ÚMultiTensorApplyFc                 óŽ   — 	 dt           _        || _        d S # t          $ r#}dt           _        |t           _        Y d }~d S d }~ww xY w)NTF)r	   Ú	availableÚ
chunk_sizeÚImportErrorÚ
import_err)Úselfr   Úerrs      ú_/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/optim/adan.pyÚ__init__zMultiTensorApply.__init__"   s^   € ð	.Ø)-ÕÔ&Ø(ˆDŒOˆOˆOøÝð 	.ð 	.ð 	.Ø).ÕÔ&Ø*-ÕÔ'Ð'Ð'Ð'Ð'Ð'Ð'øøøøð	.øøøs   ‚ —
A¡?¿Ac                 ó"   —  || j         ||g|¢R Ž S ©N)r   )r   ÚopÚnoop_flag_bufferÚtensor_listsÚargss        r   Ú__call__zMultiTensorApply.__call__*   s!   € Øˆr�$”/Ð#3°\ÐIÀDÐIÐIÐIÐIó    N)Ú__name__Ú
__module__Ú__qualname__r   Úwarnedr   r   © r   r   r	   r	      sA   € € € € € Ø€IØ€Fð.ð .ð .ðJð Jð Jð Jð Jr   r	   c                   óÚ   ‡ — e Zd ZdZ	 	 	 	 	 	 	 dded	eeeef         d
ededededee         fˆ fd„Zˆ fd„Z	 e
j        ¦   «         d„ ¦   «         Z e
j        ¦   «         dd„¦   «         Zˆ xZS )ÚAdanaØ   Implements a pytorch variant of Adan.

    Adan was proposed in Adan: Adaptive Nesterov Momentum Algorithm for Faster Optimizing Deep Models
    https://arxiv.org/abs/2208.06677

    Arguments:
        params: Iterable of parameters to optimize or dicts defining parameter groups.
        lr: Learning rate.
        betas: Coefficients used for first- and second-order moments.
        eps: Term added to the denominator to improve numerical stability.
        weight_decay: Decoupled weight decay (L2 penalty)
        no_prox: How to perform the weight decay
        caution: Enable caution from 'Cautious Optimizers'
        foreach: If True would use torch._foreach implementation. Faster but uses slightly more memory.
    çü©ñÒMbP?©g\�Âõ(\ï?gq=
×£pí?g®Gáz®ï?ç:Œ0âŽyE>ç        FNÚlrÚbetasÚepsÚweight_decayÚno_proxÚcautionÚforeachc	           	      ó†  •— d|k    s"t          d                     |¦  «        ¦  «        ‚d|k    s"t          d                     |¦  «        ¦  «        ‚d|d         cxk    rdk     s*n t          d                     |d         ¦  «        ¦  «        ‚d|d         cxk    rdk     s*n t          d                     |d         ¦  «        ¦  «        ‚d|d	         cxk    rdk     s*n t          d
                     |d	         ¦  «        ¦  «        ‚t          |||||||¬¦  «        }	t          ¦   «                              ||	¦  «         d S )Nr%   zInvalid learning rate: {}zInvalid epsilon value: {}r   ç      ð?z%Invalid beta parameter at index 0: {}é   z%Invalid beta parameter at index 1: {}é   z%Invalid beta parameter at index 2: {})r&   r'   r(   r)   r*   r+   r,   )Ú
ValueErrorÚformatÚdictÚsuperr   )r   Úparamsr&   r'   r(   r)   r*   r+   r,   ÚdefaultsÚ	__class__s             €r   r   zAdan.__init__?   sk  ø€ ð �bŠyˆyÝÐ8×?Ò?ÀÑCÔCÑDÔDÐDØ�cŠzˆzÝÐ8×?Ò?ÀÑDÔDÑEÔEÐEØ�e˜A”hÐ$Ð$Ò$Ð$ Ò$Ð$Ð$Ð$ÝÐD×KÒKÈEÐRSÌHÑUÔUÑVÔVÐVØ�e˜A”hÐ$Ð$Ò$Ð$ Ò$Ð$Ð$Ð$ÝÐD×KÒKÈEÐRSÌHÑUÔUÑVÔVÐVØ�e˜A”hÐ$Ð$Ò$Ð$ Ò$Ð$Ð$Ð$ÝÐD×KÒKÈEÐRSÌHÑUÔUÑVÔVÐVåØØØØ%ØØØð
ñ 
ô 
ˆõ 	‰Œ×Ò˜ Ñ*Ô*Ð*Ð*Ð*r   c                 óÄ   •— t          t          | ¦  «                             |¦  «         | j        D ].}|                     dd¦  «         |                     dd¦  «         Œ/d S )Nr*   Fr+   )r4   r!   Ú__setstate__Úparam_groupsÚ
setdefault)r   ÚstateÚgroupr7   s      €r   r9   zAdan.__setstate___   sl   ø€ Ý�d�DÑÔ×&Ò& uÑ-Ô-Ð-ØÔ&ð 	/ð 	/ˆEØ×Ò˜Y¨Ñ.Ô.Ð.Ø×Ò˜Y¨Ñ.Ô.Ð.Ð.ð	/ð 	/r   c                 óì   — | j         D ]k}d|d<   |d         D ][}|j        rR| j        |         }t          j        |¦  «        |d<   t          j        |¦  «        |d<   t          j        |¦  «        |d<   Œ\Œld S )Nr   Ústepr5   Úexp_avgÚ
exp_avg_sqÚexp_avg_diff)r:   Úrequires_gradr<   ÚtorchÚ
zeros_like)r   r=   Úpr<   s       r   Úrestart_optzAdan.restart_opte   sž   € àÔ&ð 	@ð 	@ˆEØˆE�&‰MØ˜8”_ð 
@ð 
@�Ø”?ð 	@Ø œJ qœM�Eõ (-Ô'7¸Ñ':Ô':�E˜)Ñ$å*/Ô*:¸1Ñ*=Ô*=�E˜,Ñ'å,1Ô,<¸QÑ,?Ô,?�E˜.Ñ)øð
@ð	@ð 	@r   c                 ó:  — d}|�5t          j        ¦   «         5   |¦   «         }ddd¦  «         n# 1 swxY w Y   	 dt           j        j        j                             ¦   «         v }n# t          $ r d}Y nw xY w| j        D �]}g }g }g }g }g }	g }
|d         \  }}}d|v r|dxx         dz  cc<   nd|d<   d||d         z  z
  }d||d         z  z
  }d||d         z  z
  }|d         D �]8}|j        €Œ| 	                    |¦  «         | 	                    |j        ¦  «         | j
        |         }t          |¦  «        d	k    rEt          j        |¦  «        |d
<   t          j        |¦  «        |d<   t          j        |¦  «        |d<   d|vs|d         dk    r|j                             ¦   «          |d<   | 	                    |d
         ¦  «         | 	                    |d         ¦  «         |	 	                    |d         ¦  «         |
 	                    |d         ¦  «         �Œ:|s�Œ¥|d         €|d          p|}n|d         }|rt          }nt          } |||f|||	|
|||||t!          j        |¦  «        |d         |d         |d         |d         |d         dœŽ �Œ|S )z$Performs a single optimization step.NÚScalarFr'   r?   r/   r.   r5   r   r@   rA   rB   Úneg_pre_gradr,   r+   r&   r)   r(   r*   )Úexp_avgsÚexp_avg_sqsÚexp_avg_diffsÚneg_pre_gradsÚbeta1Úbeta2Úbeta3Úbias_correction1Úbias_correction2Úbias_correction3_sqrtr&   r)   r(   r*   r+   )rD   Úenable_gradÚopsÚatenÚ_foreach_maximum_Ú	overloadsÚ	Exceptionr:   ÚgradÚappendr<   ÚlenrE   ÚcloneÚ_multi_tensor_adanÚ_single_tensor_adanÚmathÚsqrt)r   ÚclosureÚlossÚhas_scalar_maximumr=   Úparams_with_gradÚgradsrK   rL   rM   rN   rO   rP   rQ   rR   rS   Úbias_correction3rF   r<   Úuse_foreachÚfuncs                        r   r?   z	Adan.stepu   st  € ð ˆØÐÝÔ"Ñ$Ô$ð !ð !Ø�w‘y”y�ð!ð !ð !ñ !ô !ð !ð !ð !ð !ð !ð !øøøð !ð !ð !ð !ð	'Ø!)­U¬Y¬^Ô-M×-WÒ-WÑ-YÔ-YÐ!YÐÐøÝð 	'ð 	'ð 	'Ø!&ÐÐÐð	'øøøð Ô&ð G	ñ G	ˆEØ!ÐØˆEØˆHØˆKØˆMØˆMà"'¨¤.ÑˆE�5˜%ð ˜ˆˆØ�f��” Ñ"��‘�à !��f‘à" U¨e°F¬mÑ%;Ñ;ÐØ" U¨e°F¬mÑ%;Ñ;ÐØ" U¨e°F¬mÑ%;Ñ;Ðà˜8”_ð <ñ <�Ø”6�>ØØ ×'Ò'¨Ñ*Ô*Ð*Ø—’˜QœVÑ$Ô$Ð$àœ
 1œ�Ý�u‘:”: ’?�?Ý',Ô'7¸Ñ':Ô':�E˜)Ñ$Ý*/Ô*:¸1Ñ*=Ô*=�E˜,Ñ'Ý,1Ô,<¸QÑ,?Ô,?�E˜.Ñ)à!¨Ð.Ð.°%¸´-À1Ò2DÐ2DØ-.¬V¯\ª\©^¬^¨O�E˜.Ñ)à—’  iÔ 0Ñ1Ô1Ð1Ø×"Ò" 5¨Ô#6Ñ7Ô7Ð7Ø×$Ò$ U¨>Ô%:Ñ;Ô;Ð;Ø×$Ò$ U¨>Ô%:Ñ;Ô;Ð;Ñ;à#ð Ùà�YÔÐ'Ø"'¨	Ô"2Ð2ÐHÐ6H��à# IÔ.�àð +Ý)��å*�àˆDØ Øðð "Ø'Ø+Ø+ØØØØ!1Ø!1Ý&*¤iÐ0@Ñ&AÔ&AØ˜”;Ø" >Ô2Ø˜%”LØ˜iÔ(Ø˜iÔ(ð#ð ð ð ñ ð( ˆs   ˜/¯3¶3»*A& Á&A5Á4A5)r"   r#   r$   r%   FFNr   )r   r   r   Ú__doc__Úfloatr   Úboolr   r   r9   rD   Úno_gradrG   r?   Ú__classcell__)r7   s   @r   r!   r!   .   s'  ø€ € € € € ðð ð$ Ø0BØØ"%Ø!Ø!Ø&*ð+ð +àð+ð ˜  uÐ,Ô-ð+ð ð	+ð
  ð+ð ð+ð ð+ð ˜d”^ð+ð +ð +ð +ð +ð +ð@/ð /ð /ð /ð /ð €U„]�_„_ð@ð @ñ „_ð@ð €U„]�_„_ðUð Uð Uñ „_ðUð Uð Uð Uð Ur   r!   r5   rg   rK   rL   rM   rN   rO   rP   rQ   rR   rS   rT   r&   r)   r(   r*   r+   c                óÎ  — t          | ¦  «        D �]S\  }}||         }||         }||         }||         }||         }|                     |¦  «         |                     |¦  «                             |d|z
  ¬¦  «         |                     |¦  «                             |d|z
  ¬¦  «         |                     |¦  «                             |¦  «         |                     |¦  «                             ||d|z
  ¬¦  «         |                     ¦   «         |z                       |¦  «        }||z  |
z  }||	z  }|ra||z  dk                         |j        ¦  «        }|                     |                     ¦   «          	                    d¬¦  «        ¦  «         ||z  }|rN|                     d||z  z
  ¦  «         | 
                    ||| ¬¦  «         | 
                    ||| ¬¦  «         nM| 
                    ||| ¬¦  «         | 
                    ||| ¬¦  «         |                     d||z  z   ¦  «         |                     ¦   «                              |d¬¦  «         �ŒUd S )Nr/   ©Úalpha©Úvaluer   r"   )Úminç      ð¿)Ú	enumerateÚadd_Úmul_Úaddcmul_rb   ÚtoÚdtypeÚdiv_ÚmeanÚclamp_Úaddcdiv_Úzero_)r5   rg   rK   rL   rM   rN   rO   rP   rQ   rR   rS   rT   r&   r)   r(   r*   r+   ÚiÚparamr[   r@   rA   rB   Úneg_grad_or_diffÚdenomÚstep_size_diffÚ	step_sizeÚmasks                               r   r`   r`   Î   s�  € õ( ˜fÑ%Ô%ð #8ñ #8‰ˆˆ5Ø�QŒxˆØ˜1”+ˆØ  ”^ˆ
Ø$ QÔ'ˆØ(¨Ô+Ðð 	×Ò˜dÑ#Ô#Ð#à�Š�UÑÔ× Ò  ¨Q°©YÐ Ñ7Ô7Ð7Ø×Ò˜%Ñ Ô ×%Ò%Ð&6¸aÀ%¹iÐ%ÑHÔHÐHà×Ò˜eÑ$Ô$×)Ò)¨$Ñ/Ô/Ð/Ø�Š˜ÑÔ×'Ò'Ð(8Ð:JÐRSÐV[ÑR[Ð'Ñ\Ô\Ð\à—’Ñ"Ô"Ð%:Ñ:×@Ò@ÀÑEÔEˆØ˜e™Ð&6Ñ6ˆØÐ)Ñ)ˆ	àð 	%à˜d‘N QÒ&×*Ò*¨4¬:Ñ6Ô6ˆDØ�IŠI�d—i’i‘k”k×(Ò(¨TÐ(Ñ2Ô2Ñ3Ô3Ð3Ø ‘nˆGàð 	.Ø�JŠJ�q˜2 Ñ,Ñ,Ñ-Ô-Ð-Ø�NŠN˜7 E°)°ˆNÑ<Ô<Ð<Ø�NŠN˜<¨°~°oˆNÑFÔFÐFÐFà�NŠN˜7 E°)°ˆNÑ<Ô<Ð<Ø�NŠN˜<¨°~°oˆNÑFÔFÐFØ�JŠJ�q˜2 Ñ,Ñ,Ñ-Ô-Ð-à×ÒÑ Ô ×%Ò% d°$Ð%Ñ7Ô7Ð7Ñ7ðG#8ð #8r   c                óè  — t          | ¦  «        dk    rd S t          j        ||¦  «         t          j        ||¦  «         t          j        ||d|z
  ¬¦  «         t          j        ||¦  «         t          j        ||d|z
  ¬¦  «         t          j        ||¦  «         t          j        ||¦  «         t          j        ||¦  «         t          j        |||d|z
  ¬¦  «         t          j        |¦  «        }t          j        ||¦  «         t          j        ||¦  «         ||z  |
z  }||	z  }|rzt          j        ||¦  «        }d„ t          ||¦  «        D ¦   «         }d„ |D ¦   «         }t          j	        |d¦  «         t          j        ||¦  «         t          j        ||¦  «        }|rNt          j        | d||z  z
  ¦  «         t          j
        | ||| ¬¦  «         t          j
        | ||| ¬¦  «         nMt          j
        | ||| ¬¦  «         t          j
        | ||| ¬¦  «         t          j        | d||z  z   ¦  «         t          j        |¦  «         t          j        ||d¬¦  «         d S )	Nr   r/   rq   rs   c                 óP   — g | ]#\  }}|d k                          |j        ¦  «        ‘Œ$S )r   )r{   r|   )Ú.0ÚmÚgs      r   ú
<listcomp>z&_multi_tensor_adan.<locals>.<listcomp>7  s.   € ÐCÐCÐC©¨¨A�!�a’%—’˜AœGÑ$Ô$ÐCÐCÐCr   c                 ó6   — g | ]}|                      ¦   «         ‘ŒS r   )r~   )r‹   rŒ   s     r   rŽ   z&_multi_tensor_adan.<locals>.<listcomp>8  s    € Ð.Ð.Ð. 1�a—f’f‘h”hÐ.Ð.Ð.r   r"   rv   )r]   rD   Ú_foreach_add_Ú_foreach_mul_Ú_foreach_addcmul_Ú_foreach_sqrtÚ_foreach_div_Ú_foreach_mulÚziprX   Ú_foreach_addcdiv_Ú_foreach_zero_)r5   rg   rK   rL   rM   rN   rO   rP   rQ   rR   rS   rT   r&   r)   r(   r*   r+   r…   r†   r‡   ÚmasksÚ
mask_scales                         r   r_   r_     sœ  € õ( ˆ6�{„{�aÒÐØˆõ 
Ô˜ uÑ-Ô-Ð-å	Ô˜ %Ñ(Ô(Ð(Ý	Ô˜ %¨q°5©yÐ9Ñ9Ô9Ð9å	Ô˜ uÑ-Ô-Ð-Ý	Ô˜ }¸AÀ¹IÐFÑFÔFÐFå	Ô˜ uÑ-Ô-Ð-Ý	Ô˜ uÑ-Ô-Ð-Ý	Ô˜ UÑ+Ô+Ð+Ý	Ô˜K¨¸ÈQÐQVÉYÐWÑWÔWÐWåÔ Ñ,Ô,€EÝ	Ô˜Ð4Ñ5Ô5Ð5Ý	Ô˜˜sÑ#Ô#Ð#à˜%‘ZÐ"2Ñ2€NØÐ%Ñ%€Iàð 7åÔ" 8¨UÑ3Ô3ˆØCÐCµ°U¸EÑ1BÔ1BÐCÑCÔCˆØ.Ð.¨Ð.Ñ.Ô.ˆ
ÝÔ 
¨DÑ1Ô1Ð1ÝÔ˜E :Ñ.Ô.Ð.ÝÔ% h°Ñ6Ô6ˆàð ;ÝÔ˜F A¨¨\Ñ(9Ñ$9Ñ:Ô:Ð:ÝÔ ¨°%À	¸zÐJÑJÔJÐJÝÔ ¨°uÀ^ÀOÐTÑTÔTÐTÐTåÔ ¨°%À	¸zÐJÑJÔJÐJÝÔ ¨°uÀ^ÀOÐTÑTÔTÐTÝÔ˜F A¨¨\Ñ(9Ñ$9Ñ:Ô:Ð:å	Ô˜Ñ'Ô'Ð'Ý	Ô˜ u°DÐ9Ñ9Ô9Ð9Ð9Ð9r   )rk   ra   Útypingr   r   r   rD   r   Útorch.optim.optimizerr   Úobjectr	   r!   rl   rm   r`   r_   r   r   r   ú<module>rž      sy  ððð ð* €€€Ø (Ð (Ð (Ð (Ð (Ð (Ð (Ð (Ð (Ð (à €€€Ø Ð Ð Ð Ð Ð Ø +Ð +Ð +Ð +Ð +Ð +ðJð Jð Jð Jð J�vñ Jô Jð Jð ]ð ]ð ]ð ]ð ]ˆ9ñ ]ô ]ð ]ð@78Ø�V”ð78à�FŒ|ð78ð �v”,ð78ð ˜&”\ð	78ð
 ˜F”|ð78ð ˜F”|ð78ð ð78ð ð78ð ð78ð  ð78ð  ð78ð  %ð78ð ð78ð ð78ð  ð!78ð" ð#78ð$ ð%78ð 78ð 78ð 78ðt?:Ø�V”ð?:à�FŒ|ð?:ð �v”,ð?:ð ˜&”\ð	?:ð
 ˜F”|ð?:ð ˜F”|ð?:ð ð?:ð ð?:ð ð?:ð  ð?:ð  ð?:ð  %ð?:ð ð?:ð ð?:ð  ð!?:ð" ð#?:ð$ ð%?:ð ?:ð ?:ð ?:ð ?:ð ?:r   