§
    ·¯¸júL  ã            +       ó*  — d Z ddlmZmZmZmZmZ ddlZddlmZ ddl	m
Z
 ddlmZ dd	gZd
„ Zd(d„Zd„ Zd„ Z G d„ de
¦  «        Zdee         dee         dee         dee         dee         dee         dee         dedededeeef         dedee         dee         dededed ed!ed"ef(d#„Zdee         dee         dee         dee         dee         dee         dee         dedededeeef         dedee         dee         dededed ed!ed"ef(d$„Z	 	 	 	 	 	 d)dee         dee         dee         dee         dee         d&ee         d!ed"edee         dee         dedededeeef         dedee         dee         dededed ef*d'„ZdS )*u@   ADOPT PyTorch Optimizer

ADOPT: Modified Adam Can Converge with Any Î²2 with the Optimal Rate: https://arxiv.org/abs/2411.02853

Modified for reduced dependencies on PyTorch internals from original at: https://github.com/iShohei220/adopt

@inproceedings{taniguchi2024adopt,
 author={Taniguchi, Shohei and Harada, Keno and Minegishi, Gouki and Oshima, Yuta and Jeong, Seong Cheol and Nagahara, Go and Iiyama, Tomoshi and Suzuki, Masahiro and Iwasawa, Yusuke and Matsuo, Yutaka},
 booktitle = {Advances in Neural Information Processing Systems},
 title = {ADOPT: Modified Adam Can Converge with Any Î²2 with the Optimal Rate},
 year = {2024}
}

References for added functionality:
    Cautious Optimizers: https://arxiv.org/abs/2411.16085
    Why Gradients Rapidly Increase Near the End of Training: https://arxiv.org/abs/2506.02285
é    )ÚcastÚListÚOptionalÚTupleÚUnionN)ÚTensor)Ú	Optimizeré   )ÚParamsTÚAdoptÚadoptc                 óÖ   — t          | ¦  «        D ]X\  }}t          j        |¦  «        r?t          j        | |         ¦  «        | |<   |D ]}t          j        ||         ¦  «        ||<   Œ ŒYd S ©N)Ú	enumerateÚtorchÚ
is_complexÚview_as_real)ÚparamsÚstate_and_gradsÚiÚpÚss        ú`/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/optim/adopt.pyÚ_view_as_realr      s|   € Ý˜&Ñ!Ô!ð 0ð 0‰ˆˆ1ÝÔ˜AÑÔð 	0ÝÔ*¨6°!¬9Ñ5Ô5ˆF�1‰IØ$ð 0ð 0�ÝÔ)¨!¨A¬$Ñ/Ô/��!‘�øð	0ð 0ó    c                 ó�   — | rt           j        S t          j        ¦   «         t           j        k    rt           j        nt           j        S r   )r   Úfloat32Úget_default_dtypeÚfloat64)Úis_fuseds    r   Ú_get_scalar_dtyper!   $   s7   € Øð ÝŒ}ÐåÔ0Ñ2Ô2µe´mÒCÐC�ŒˆÍÌðr   c                  ó    — t          t          d¦  «        r8t          t          j        d¦  «        rt          j                             ¦   «         S dS )NÚcompilerÚis_compilingF)Úhasattrr   r#   r$   © r   r   Ú_is_compilingr'   ,   s@   € Ý�u�jÑ!Ô!ð ¥g­e¬n¸nÑ&MÔ&Mð ÝŒ~×*Ò*Ñ,Ô,Ð,àˆur   c                 ó¾   — t           j                             ¦   «         st          ¦   «         r| S t	          | t           j        ¦  «        r|                      ¦   «         n| S r   )r   ÚjitÚis_scriptingr'   Ú
isinstancer   Úitem)Úxs    r   Ú
_get_valuer.   3   sK   € åŒ9×!Ò!Ñ#Ô#ð >­©¬ð >Øˆå% a­¬Ñ6Ô6Ð=ˆq�vŠv‰xŒxˆx¸AÐ=r   c                   óð   ‡ — e Zd ZdZ	 	 	 	 	 	 	 dddddddœd	ed
eeef         deeef         dede	e         dede
de
de
de	e
         de
de
de
fˆ fd„Zˆ fd„Zd„ Z ej        ¦   «         dd„¦   «         Zˆ xZS )r   uq   
    ADOPT: Modified Adam Can Converge with Any Î²2 with the Optimal Rate: https://arxiv.org/abs/2411.02853

    çü©ñÒMbP?©gÍÌÌÌÌÌì?g§èH.ÿï?ç�íµ ÷Æ°>çZd;ßOÕ?ç        F)ÚcautionÚforeachÚmaximizeÚ
capturableÚdifferentiabler   ÚlrÚbetasÚepsÚclip_expÚweight_decayÚ	decoupledÚcorrected_weight_decayr5   r6   r7   r8   r9   c	                ód  •— t          |t          ¦  «        r:|
r|st          d¦  «        ‚|                     ¦   «         dk    rt          d¦  «        ‚d|k    st          d|› �¦  «        ‚d|k    st          d|› �¦  «        ‚d|d         cxk    rdk     sn t          d	|d         › �¦  «        ‚d|d         cxk    rdk     sn t          d
|d         › �¦  «        ‚d|k    st          d|› �¦  «        ‚t	          ||||||||	||
||¬¦  «        }t          ¦   «                              ||¦  «         d S )NúElr as a Tensor is not supported for capturable=False and foreach=Truer
   zTensor lr must be 1-elementr4   zInvalid learning rate: zInvalid epsilon value: r   ç      ð?z#Invalid beta parameter at index 0: z#Invalid beta parameter at index 1: zInvalid weight_decay value: )r:   r;   r<   r>   r=   r?   r@   r5   r7   r6   r8   r9   )r+   r   Ú
ValueErrorÚnumelÚdictÚsuperÚ__init__)Úselfr   r:   r;   r<   r=   r>   r?   r@   r5   r6   r7   r8   r9   ÚdefaultsÚ	__class__s                  €r   rH   zAdopt.__init__@   s–  ø€ õ" �b�&Ñ!Ô!ð 	@Øð ˜zð Ý Ø[ñô ð ð �xŠx‰zŒz˜QŠˆÝ Ð!>Ñ?Ô?Ð?Ø�bŠyˆyÝÐ;°rÐ;Ð;Ñ<Ô<Ð<Ø�cŠzˆzÝÐ<°sÐ<Ð<Ñ=Ô=Ð=Ø�e˜A”hÐ$Ð$Ò$Ð$ Ò$Ð$Ð$Ð$ÝÐMÀ5ÈÄ8ÐMÐMÑNÔNÐNØ�e˜A”hÐ$Ð$Ò$Ð$ Ò$Ð$Ð$Ð$ÝÐMÀ5ÈÄ8ÐMÐMÑNÔNÐNØ�lÒ"Ð"ÝÐJ¸LÐJÐJÑKÔKÐKåØØØØ%ØØØ#9ØØØØ!Ø)ð
ñ 
ô 
ˆõ 	‰Œ×Ò˜ Ñ*Ô*Ð*Ð*Ð*r   c                 ó  •— t          ¦   «                              |¦  «         | j        D �]Z}|                     dd¦  «         |                     dd ¦  «         |                     dd¦  «         |                     dd¦  «         |                     dd ¦  «         |                     dd¦  «         |                     dd¦  «         |d	         D ]´}| j                             |g ¦  «        }t          |¦  «        d
k    r„t          j        |d         ¦  «        sjt          |d         ¦  «        }|d         r(t          j
        |t          ¦   «         |j        ¬¦  «        n!t          j
        |t          ¦   «         ¬¦  «        |d<   Œµ�Œ\d S )Nr7   Fr6   r8   r9   r=   r5   r@   r   r   Ústep©ÚdtypeÚdevice©rO   )rG   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetÚlenr   Ú	is_tensorÚfloatÚtensorr!   rP   )rI   rU   Úgroupr   Úp_stateÚstep_valrK   s         €r   rR   zAdopt.__setstate__s   s“  ø€ Ý‰Œ×Ò˜UÑ#Ô#Ð#ØÔ&ð 	ñ 	ˆEØ×Ò˜Z¨Ñ/Ô/Ð/Ø×Ò˜Y¨Ñ-Ô-Ð-Ø×Ò˜\¨5Ñ1Ô1Ð1Ø×ÒÐ-¨uÑ5Ô5Ð5Ø×Ò˜Z¨Ñ.Ô.Ð.Ø×Ò˜Y¨Ñ.Ô.Ð.Ø×ÒÐ5°uÑ=Ô=Ð=Ø˜8”_ð ð �Øœ*Ÿ.š.¨¨BÑ/Ô/�Ý�w‘<”< 1Ò$Ð$­U¬_¸WÀV¼_Ñ-MÔ-MÐ$Ý$ W¨V¤_Ñ5Ô5�Hð ! Ô.ðO�œØ$Ý"3Ñ"5Ô"5Ø#$¤8ðñ ô ð õ #œ\¨(Õ:KÑ:MÔ:MÐNÑNÔNð ˜F‘Oøñ	ð	ð 	r   c                 óæ  — d}|d         D �]ä}|j         €Œ|t          j        |¦  «        z  }|                     |¦  «         |j         j        rt          d¦  «        ‚|                     |j         ¦  «         | j        |         }	t          |	¦  «        dk    rª|d         r-t          j        dt          ¦   «         |j         j
        ¬¦  «        n!t          j        dt          ¦   «         ¬	¦  «        |	d
<   t          j        |j         t          j        ¬¦  «        |	d<   t          j        |j         t          j        ¬¦  «        |	d<   |                     |	d         ¦  «         |                     |	d         ¦  «         |d         r|	d
         j        rt          d¦  «        ‚|d         r1t          j        |d         ¦  «        r|d         st          d¦  «        ‚|                     |	d
         ¦  «         �Œæ|S )NFr   z'ADOPT does not support sparse gradientsr   r8   r&   rN   r4   rQ   rM   )Úmemory_formatÚexp_avgÚ
exp_avg_sqr9   zB`requires_grad` is not supported for `step` in differentiable moder6   r:   rB   )Úgradr   r   ÚappendÚ	is_sparseÚRuntimeErrorrU   rW   Úzerosr!   rP   rZ   Ú
zeros_likeÚpreserve_formatÚrequires_gradrX   )
rI   r[   Úparams_with_gradÚgradsÚexp_avgsÚexp_avg_sqsÚstate_stepsÚhas_complexr   rU   s
             r   Ú_init_groupzAdopt._init_group‹   sí  € ð ˆØ�x”ð #	.ñ #	.ˆAØŒvˆ~ØØ�5Ô+¨AÑ.Ô.Ñ.ˆKØ×#Ò# AÑ&Ô&Ð&ØŒvÔð NÝ"Ð#LÑMÔMÐMØ�LŠL˜œÑ Ô Ð à”J˜q”MˆEå�5‰zŒz˜QŠˆð ˜\Ô*ðF•E”K Õ*;Ñ*=Ô*=ÀaÄfÄmÐTÑTÔTÐTåœ cÕ1BÑ1DÔ1DÐEÑEÔEð �f‘õ $)Ô#3°A´FÍ%ÔJ_Ð#`Ñ#`Ô#`��iÑ å&+Ô&6°q´vÍUÔMbÐ&cÑ&cÔ&c��lÑ#à�OŠO˜E )Ô,Ñ-Ô-Ð-Ø×Ò˜u \Ô2Ñ3Ô3Ð3àÐ%Ô&ð i¨5°¬=Ô+Fð iÝ"Ð#gÑhÔhÐhð �YÔð l¥E¤O°E¸$´KÑ$@Ô$@ð lÈÈ|ÔI\ð lÝ"Ð#jÑkÔkÐkà×Ò˜u Vœ}Ñ-Ô-Ð-Ñ-ØÐr   Nc                 óØ  — t          | d¦  «        r|                      ¦   «          n$t          | d¦  «        r|                      ¦   «          d}|�5t          j        ¦   «         5   |¦   «         }ddd¦  «         n# 1 swxY w Y   | j        D ]ß}g }g }g }g }g }|d         \  }	}
|                      ||||||¦  «        }t          |||||fi d|“d|	“d|
“d|d         “d	|d	         “d
|d
         “d|d         r| j        d         nd“d|d         “d|d         “d|d         “d|d         “d|d         “d|d         “d|d         “dt          | dd¦  «        “dt          | dd¦  «        “Ž Œà|S )z°Perform a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Ú'_accelerator_graph_capture_health_checkÚ _cuda_graph_capture_health_checkNr;   ro   Úbeta1Úbeta2r:   r>   r=   Úmax_lrr@   r?   r<   r5   r7   r6   r8   r9   Ú
grad_scaleÚ	found_inf)
r%   rr   rs   r   Úenable_gradrS   rp   r   rJ   Úgetattr)rI   ÚclosureÚlossr[   rj   rk   rl   rm   rn   rt   ru   ro   s               r   rM   z
Adopt.step¼   s‹  € õ �4ÐBÑCÔCð 	4Ø×8Ò8Ñ:Ô:Ð:Ð:Ý�TÐ=Ñ>Ô>ð 	4Ø×1Ò1Ñ3Ô3Ð3àˆØÐÝÔ"Ñ$Ô$ð !ð !Ø�w‘y”y�ð!ð !ð !ñ !ô !ð !ð !ð !ð !ð !ð !øøøð !ð !ð !ð !ð Ô&ð '	ð '	ˆEØ-/ÐØ"$ˆEØ%'ˆHØ(*ˆKØ(*ˆKØ  œ>‰LˆE�5à×*Ò*ØØ ØØØØñô ˆKõ Ø ØØØØðð ð ð (˜Kðð �eðð �eðð ˜”;�;ðð # >Ô2Ð2ðð ˜zÔ*Ð*ðð /4Ð4LÔ.MÐW�t”} TÔ*Ð*ÐSWøðð   Ô,Ð,ðð ˜%”L�Lðð ˜iÔ(Ð(ðð  ˜zÔ*Ð*ð!ð" ˜iÔ(Ð(ð#ð$ ! Ô.Ð.ð%ð&  %Ð%5Ô6Ð6ð'õ( # 4¨°tÑ<Ô<Ð<ð)õ* " $¨°TÑ:Ô:Ð:ð+ð ð ð0 ˆs   Á!A8Á8A<Á?A<)r0   r1   r2   r3   r4   FFr   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   rY   r   r   r   ÚboolrH   rR   rp   r   Úno_gradrM   Ú__classcell__)rK   s   @r   r   r   ;   sj  ø€ € € € € ðð ð (,Ø)6ØØ(-Ø"%Ø#Ø+0ð1+ð "Ø&+Ø"Ø$Ø#(ð1+ð 1+ð 1+àð1+ð �e˜V�mÔ$ð1+ð ˜ ˜Ô&ð	1+ð
 ð1+ð ˜u”oð1+ð  ð1+ð ð1+ð %)ð1+ð ð1+ð ˜d”^ð1+ð ð1+ð ð1+ð !ð1+ð 1+ð 1+ð 1+ð 1+ð 1+ðfð ð ð ð ð0.ð .ð .ðb €U„]�_„_ð:ð :ð :ñ „_ð:ð :ð :ð :ð :r   r   rk   rl   rm   rn   rw   rx   ro   rt   ru   r:   r>   r=   rv   r?   r<   r5   r7   r8   r9   c                ó  — |€|�J ‚t           j                             ¦   «         rt          |
t          ¦  «        sJ ‚t          | ¦  «        D �]¸\  }}|s||         n||          }||         }||         }||         }|rTt          ¦   «         sFddlm}  |¦   «         }|j	        j
        |j	        j
        k    r|j	        j
        |v sJ d|› d�¦   «         ‚|dz  }t          j        |¦  «        rTt          j        |¦  «        }|�t          j        |¦  «        }|�t          j        |¦  «        }t          j        |¦  «        }|dk    r|s|                     ||¬¦  «        }|s|r|nt          |¦  «        }|dk    r*|                     ||                     ¦   «         ¦  «         �ŒX|dk    r)|r'|�|
dz  |z  n|
}|                     || |z  ¬¦  «         t          j        |                     ¦   «         |¦  «        }|                     |¦  «        }|�|dz
  |z  } |                     |  | ¦  «         |                     |d|z
  ¦  «         |ra||z  dk                         |j        ¦  «        }!|!                     |!                     ¦   «                              d¬	¦  «        ¦  «         ||!z  }|                     ||
 ¬¦  «         |                     |	¦  «                             ||                     ¦   «         d|	z
  ¬
¦  «         �Œºd S )Nr   ©Ú!_get_capturable_supported_devicesúIIf capturable=True, params and state_steps must be on supported devices: ú.r
   ©Úalphaé   r0   )Úmin©Úvalue)r   r)   r*   r+   rY   r   r'   Útorch.optim.optimizerr†   rP   Útyper   r   Úaddr.   Úaddcmul_ÚconjÚadd_ÚclampÚsqrtÚdivÚclamp_Úlerp_ÚtorO   Údiv_ÚmeanÚmul_)"r   rk   rl   rm   rn   rw   rx   ro   rt   ru   r:   r>   r=   rv   r?   r<   r5   r7   r8   r9   r   Úparamrb   r`   ra   Ústep_tr†   Úcapturable_supported_devicesrM   Úwd_scaleÚdenomÚnormed_gradÚclip_valÚmasks"                                     r   Ú_single_tensor_adoptr¦   ú   sK  € ð. Ð )Ð"3Ð"3Ð3å„y×ÒÑÔð %õ ˜"�eÑ$Ô$Ð$Ð$Ð$å˜fÑ%Ô%ð 5Lñ 5L‰ˆˆ5Ø'Ð6ˆu�QŒxˆx¨e°A¬h¨YˆØ˜1”+ˆØ  ”^ˆ
Ø˜Q”ˆð ð 	|�m™oœoð 	|ØOÐOÐOÐOÐOÐOØ+LÐ+LÑ+NÔ+NÐ(Ø”<Ô$¨¬Ô(:Ò:Ð:¸u¼|Ô?PÐTpÐ?pÐ?pÐ?pØ{Ð\xÐ{Ð{Ð{ñ @qÔ?pÐpð 	�!‰ˆåÔ˜EÑ"Ô"ð 	.ÝÔ% dÑ+Ô+ˆDØÐ"ÝÔ,¨WÑ5Ô5�ØÐ%Ý"Ô/°
Ñ;Ô;�
ÝÔ& uÑ-Ô-ˆEà˜1ÒÐ YÐØ—8’8˜E¨�8Ñ6Ô6ˆDà#ÐM ~ÐMˆvˆv½:ÀfÑ;MÔ;MˆØ�1Š9ˆ9Ø×Ò  d§i¢i¡k¤kÑ2Ô2Ð2Ùà˜1ÒÐ ÐØ+1Ð+=�r˜Q‘w Ñ'Ð'À2ˆHØ�JŠJ�u X I°Ñ$<ˆJÑ=Ô=Ð=å”˜JŸOšOÑ-Ô-¨sÑ3Ô3ˆØ—h’h˜u‘o”oˆàÐØ˜q™ XÑ-ˆHØ×Ò ˜y¨(Ñ3Ô3Ð3à�Š�k 1 u¡9Ñ-Ô-Ð-àð 	%à˜d‘N QÒ&×*Ò*¨4¬:Ñ6Ô6ˆDØ�IŠI�d—i’i‘k”k×(Ò(¨TÐ(Ñ2Ô2Ñ3Ô3Ð3Ø ‘nˆGà�
Š
�7 2 #ˆ
Ñ&Ô&Ð&à�Š˜ÑÔ×'Ò'¨¨d¯iªi©k¬kÀÀUÁÐ'ÑKÔKÐKÑKðk5Lð 5Lr   c                óÚ  ‡'— t          | ¦  «        dk    rd S t          |
t          ¦  «        r|st          d¦  «        ‚|rWt	          ¦   «         sIddlm}  |d¬¦  «        Š't          ˆ'fd„t          | |¦  «        D ¦   «         ¦  «        sJ d‰'› d�¦   «         ‚|€|�J ‚|r
J d	¦   «         ‚t          j
        | ||||g¦  «        }|                     ¦   «         D �]\  \  }}}}}}t          t          t                   |¦  «        }t          t          t                   |¦  «        }t          t          t                   |¦  «        }t          t          t                   |¦  «        }t          t          t                   |¦  «        } |rt          ||||¦  «         |rt          j        |¦  «        }t	          ¦   «         s9| d         j        r,t          j        | t          j        d
d¬¦  «        d
¬¦  «         nt          j        | d¦  «         |dk    r3|s1|rt          j        |||¬¦  «         nt          j        |||¬¦  «        }| d         dk    rt          j        |||¦  «         �Œ�|dk    r)|r'|�|
dz  |z  n|
}!t          j        |||! |z  ¬¦  «         t          j        |¦  «        }"t          j        |"|¦  «         t          j        ||"¦  «        }#|�9| d         dz
  |z  }$t          j        |#|$ ¦  «         t          j        |#|$¦  «         t          j        ||#d|z
  ¦  «         |rzt          j        ||¦  «        }%d„ t          |%|¦  «        D ¦   «         }%d„ |%D ¦   «         }&t          j        |&d¦  «         t          j        |%|&¦  «         t          j        ||%¦  «        }t          j        |||
 ¬¦  «         t          j        ||	¦  «         t          j        |||d|	z
  ¬¦  «         �Œd S )Nr   rB   r…   F)Úsupports_xlac              3   ón   •K  — | ]/\  }}|j         j        |j         j        k    o|j         j        ‰v V — Œ0d S r   )rP   r�   )Ú.0r   rM   r    s      €r   ú	<genexpr>z&_multi_tensor_adopt.<locals>.<genexpr>v  sX   øè è € ð 
ð 
á��4ð ŒHŒM˜Tœ[Ô-Ò-Ð_°!´(´-ÐC_Ð2_ð
ð 
ð 
ð 
ð 
ð 
r   r‡   rˆ   z#_foreach ops don't support autogradrC   Úcpu)rP   r‰   r
   r‹   c                 óP   — g | ]#\  }}|d k                          |j        ¦  «        ‘Œ$S )r   )rš   rO   )rª   ÚmÚgs      r   ú
<listcomp>z'_multi_tensor_adopt.<locals>.<listcomp>½  s.   € ÐNÐNÐN©T¨Q°�a˜!’e—Z’Z ¤Ñ(Ô(ÐNÐNÐNr   c                 ó6   — g | ]}|                      ¦   «         ‘ŒS r&   )rœ   )rª   r®   s     r   r°   z'_multi_tensor_adopt.<locals>.<listcomp>¾  s    € Ð2Ð2Ð2 q˜!Ÿ&š&™(œ(Ð2Ð2Ð2r   r0   r�   )rW   r+   r   re   r'   r�   r†   ÚallÚzipr	   Ú"_group_tensors_by_device_and_dtypeÚvaluesr   r   r   r   Ú_foreach_negÚis_cpuÚ_foreach_add_rZ   Ú_foreach_addÚ_foreach_addcmul_Ú_foreach_sqrtÚ_foreach_maximum_Ú_foreach_divÚ_foreach_minimum_Ú_foreach_lerp_Ú_foreach_mulÚ_foreach_div_Ú_foreach_mul_)(r   rk   rl   rm   rn   rw   rx   ro   rt   ru   r:   r>   r=   rv   r?   r<   r5   r7   r8   r9   r†   Úgrouped_tensorsÚdevice_params_Údevice_grads_Údevice_exp_avgs_Údevice_exp_avg_sqs_Údevice_state_steps_Ú_Údevice_paramsÚdevice_gradsÚdevice_exp_avgsÚdevice_exp_avg_sqsÚdevice_state_stepsr¡   Úexp_avg_sq_sqrtr£   r¤   ÚmasksÚ
mask_scaler    s(                                          @r   Ú_multi_tensor_adoptrÒ   Q  s«  ø€ õ. ˆ6�{„{�aÒÐØˆå�"•fÑÔð 
 jð 
ÝØSñ
ô 
ð 	
ð
 ð w�-™/œ/ð wØKÐKÐKÐKÐKÐKØ'HÐ'HØð(
ñ (
ô (
Ð$õ ð 
ð 
ð 
ð 
å˜v {Ñ3Ô3ð
ñ 
ô 
ñ 
ô 
ð 	wð 	wð wÐWsÐvÐvÐvñ	wô 	wð 
ð
 Ð )Ð"3Ð"3Ð3àÐDÐDÐDÑDÔDÐåÔBØ	�˜ +¨{Ð;ñô €Oð ×"Ò"Ñ$Ô$ðDañ Dañ 		ñ 	ØØØØØØÝ�T¥&œ\¨>Ñ:Ô:ˆÝ�D¥œL¨-Ñ8Ô8ˆÝ�t¥Fœ|Ð-=Ñ>Ô>ˆÝ!¥$¥v¤,Ð0CÑDÔDÐÝ!¥$¥v¤,Ð0CÑDÔDÐð ð 	\Ý˜-¨°ÐHZÑ[Ô[Ð[àð 	<Ý Ô-¨lÑ;Ô;ˆLõ ‰Œð 	7Ð#5°aÔ#8Ô#?ð 	7ÝÔÐ 2µE´LÀÈUÐ4SÑ4SÔ4SÐ[^Ð_Ñ_Ô_Ð_Ð_åÔÐ 2°AÑ6Ô6Ð6à˜1ÒÐ YÐàð cÝÔ# L°-À|ÐTÑTÔTÐTÐTå$Ô1°,ÀÐUaÐbÑbÔb�à˜aÔ  AÒ%Ð%ÝÔ#Ð$6¸ÀlÑSÔSÐSÙà˜1ÒÐ ÐØ+1Ð+=�r˜Q‘w Ñ'Ð'À2ˆHÝÔ ¨}ÀXÀIÐP\ÑD\Ð]Ñ]Ô]Ð]åÔ-Ð.@ÑAÔAˆÝÔ °Ñ5Ô5Ð5åÔ(¨°ÑGÔGˆàÐØ*¨1Ô-°Ñ1°hÑ>ˆHÝÔ# K°(°Ñ;Ô;Ð;ÝÔ# K°Ñ:Ô:Ð:åÔ˜_¨k¸1¸u¹9ÑEÔEÐEàð 	IåÔ& ¸ÑEÔEˆEØNÐNµS¸ÀÑ5MÔ5MÐNÑNÔNˆEØ2Ð2¨EÐ2Ñ2Ô2ˆJÝÔ# J°Ñ5Ô5Ð5ÝÔ  zÑ2Ô2Ð2Ý#Ô0°À%ÑHÔHˆOåÔ˜M¨?À2À#ÐFÑFÔFÐFåÔÐ.°Ñ6Ô6Ð6ÝÔÐ 2°LÀ,ÐVWÐZ_ÑV_Ð`Ñ`Ô`Ð`Ñ`ðIDað Dar   Fr6   c       
         ól  — |€d}t          ¦   «         s(t          d„ |D ¦   «         ¦  «        st          d¦  «        ‚|r-t          j                             ¦   «         rt          d¦  «        ‚|r&t          j                             ¦   «         st          }nt          } || ||||f|
||||||||||||||	dœŽ dS )z?Functional API that performs ADOPT algorithm computation.

    NFc              3   óJ   K  — | ]}t          |t          j        ¦  «        V — Œd S r   )r+   r   r   )rª   Úts     r   r«   zadopt.<locals>.<genexpr>ì  s.   è è € Ð&XÐ&XÀq¥z°!µU´\Ñ'BÔ'BÐ&XÐ&XÐ&XÐ&XÐ&XÐ&Xr   zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsz6torch.jit.script not supported with foreach optimizers)ro   rt   ru   r:   r>   r=   rv   r?   r<   r5   r7   r8   r9   rw   rx   )r'   r²   re   r   r)   r*   rÒ   r¦   )r   rk   rl   rm   rn   r6   r8   r9   rw   rx   ro   rt   ru   r:   r>   r=   rv   r?   r<   r5   r7   Úfuncs                         r   r   r   Ê  s  € ð: €Øˆõ ‰?Œ?ð 
¥3Ð&XÐ&XÈKÐ&XÑ&XÔ&XÑ#XÔ#Xð 
ÝØ^ñ
ô 
ð 	
ð ð U•5”9×)Ò)Ñ+Ô+ð UÝÐSÑTÔTÐTàð $•u”y×-Ò-Ñ/Ô/ð $Ý"ˆˆå#ˆà€DØØØØØðð  ØØØØ!ØØØØØØØØ%ØØð)ð ð ð ð ð r   r   )NFFNNF)r€   Útypingr   r   r   r   r   r   r   r�   r	   Ú_typesr   Ú__all__r   r!   r'   r.   r   r�   rY   r¦   rÒ   r   r&   r   r   ú<module>rÚ      s¼  ððð ð" 6Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5Ð 5à €€€Ø Ð Ð Ð Ð Ð Ø +Ð +Ð +Ð +Ð +Ð +à Ð Ð Ð Ð Ð à�GÐ
€ð0ð 0ð 0ðð ð ð ðð ð ð>ð >ð >ð|ð |ð |ð |ð |ˆIñ |ô |ð |ð~TLØ�V”ðTLà�FŒ|ðTLð �v”,ðTLð ˜&”\ð	TLð
 ˜&”\ðTLð ˜VÔ$ðTLð ˜FÔ#ðTLð ðTLð ðTLð ðTLð �%˜�-Ô ðTLð ðTLð ˜5”/ðTLð ˜”ðTLð  ð!TLð" ð#TLð$ ð%TLð& ð'TLð( ð)TLð* ð+TLð TLð TLð TLðnuaØ�V”ðuaà�FŒ|ðuað �v”,ðuað ˜&”\ð	uað
 ˜&”\ðuað ˜VÔ$ðuað ˜FÔ#ðuað ðuað ðuað ðuað �%˜�-Ô ðuað ðuað ˜5”/ðuað ˜”ðuað  ð!uað" ð#uað$ ð%uað& ð'uað( ð)uað* ð+uað uað uað uaðB #'Ø Ø$Ø'+Ø&*Ø!ðDð DØ�V”ðDà�FŒ|ðDð �v”,ðDð ˜&”\ð	Dð
 ˜&”\ðDð ˜$”ðDð ðDð ðDð ˜VÔ$ðDð ˜FÔ#ðDð ðDð ðDð  ð!Dð" �%˜�-Ô ð#Dð$ ð%Dð& ˜5”/ð'Dð( ˜”ð)Dð* ð+Dð, ð-Dð. ð/Dð0 ð1Dð Dð Dð Dð Dð Dr   