§
    ·¯¸j'#  ã                   óô  — d Z ddlmZmZmZ ddlZddlmZ ddlm	Z	m
Z
 ddlmZ  G d„ d	e¦  «        Z	 	 dddœdeej                 deej                 deej                 dedededededededee         fd„Zdeej                 deej                 deej                 dededededededee         fd„Zdeej                 deej                 deej                 dededededededee         fd„ZdS )ah   Lion Optimizer
Paper: `Symbolic Discovery of Optimization Algorithms` - https://arxiv.org/abs/2302.06675
Original Impl: https://github.com/google/automl/tree/master/lion

References for added functionality:
    Cautious Optimizers: https://arxiv.org/abs/2411.16085
    Why Gradients Rapidly Increase Near the End of Training: https://arxiv.org/abs/2506.02285
é    )ÚListÚOptionalÚTupleN)Ú	Optimizeré   )Ú_add_scaled_Ú_validate_scalar)ÚParamsTc                   ó¬   ‡ — e Zd ZdZ	 	 	 	 	 	 	 ddeded	eeef         d
ededededee         fˆ fd„Z	ˆ fd„Z
 ej        ¦   «         dd„¦   «         Zˆ xZS )ÚLionzImplements Lion algorithm.ç-Cëâ6?©gÍÌÌÌÌÌì?g®Gáz®ï?ç        FNÚparamsÚlrÚbetasÚweight_decayÚcautionÚcorrected_weight_decayÚmaximizeÚforeachc	           	      óŒ  •— t          d|¦  «         d|d         cxk    rdk     s*n t          d                     |d         ¦  «        ¦  «        ‚d|d         cxk    rdk     s*n t          d                     |d         ¦  «        ¦  «        ‚t          |||||||¬¦  «        }	t	          ¦   «                              ||	¦  «         d	S )
a¶  Initialize the hyperparameters.

        Args:
            params: iterable of parameters to optimize or dicts defining parameter groups
            lr: learning rate
            betas: coefficients used for computing running averages of gradient and its square
            weight_decay: weight decay coefficient
            caution: apply caution
            corrected_weight_decay: apply corrected weight decay (lr**2 / max_lr)
        zlearning rater   r   g      ð?z%Invalid beta parameter at index 0: {}r   z%Invalid beta parameter at index 1: {})r   r   r   r   r   r   r   N)r	   Ú
ValueErrorÚformatÚdictÚsuperÚ__init__)Úselfr   r   r   r   r   r   r   r   ÚdefaultsÚ	__class__s             €ú_/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/optim/lion.pyr   zLion.__init__#   sê   ø€ õ, 	˜¨"Ñ-Ô-Ð-Ø�e˜A”hÐ$Ð$Ò$Ð$ Ò$Ð$Ð$Ð$ÝÐD×KÒKÈEÐRSÌHÑUÔUÑVÔVÐVØ�e˜A”hÐ$Ð$Ò$Ð$ Ò$Ð$Ð$Ð$ÝÐD×KÒKÈEÐRSÌHÑUÔUÑVÔVÐVÝØØØ%ØØ#9ØØð
ñ 
ô 
ˆõ 	‰Œ×Ò˜ Ñ*Ô*Ð*Ð*Ð*ó    c                 ó  •— t          ¦   «                              |¦  «         | j        D ]Z}|                     dd¦  «         |                     dd¦  «         |                     dd¦  «         |                     dd ¦  «         Œ[d S )Nr   Fr   r   r   )r   Ú__setstate__Úparam_groupsÚ
setdefault)r   ÚstateÚgroupr    s      €r!   r$   zLion.__setstate__I   s“   ø€ Ý‰Œ×Ò˜UÑ#Ô#Ð#ØÔ&ð 	.ð 	.ˆEØ×Ò˜Y¨Ñ.Ô.Ð.Ø×ÒÐ5°uÑ=Ô=Ð=Ø×Ò˜Z¨Ñ/Ô/Ð/Ø×Ò˜Y¨Ñ-Ô-Ð-Ð-ð		.ð 	.r"   c                 óÂ  — d}|�5t          j        ¦   «         5   |¦   «         }ddd¦  «         n# 1 swxY w Y   | j        D �]}g }g }g }|d         \  }}|d         D ]²}	|	j        €Œ
|                     |	¦  «         |	j        j        rt          d¦  «        ‚|                     |	j        ¦  «         | j        |	         }
t          |
¦  «        dk    r#t          j	        |	t           j
        ¬¦  «        |
d<   |                     |
d         ¦  «         Œ³t          ||||||d         |d	         |d
         |d         |d         |d         r| j        d         nd¬¦  «         �Œ|S )z´Performs a single optimization step.

        Args:
            closure: A closure that reevaluates the model and returns the loss.

        Returns:
            the loss.
        Nr   r   z&Lion does not support sparse gradientsr   )Úmemory_formatÚexp_avgr   r   r   r   r   r   )Úbeta1Úbeta2r   r   r   r   r   Úmax_lr)ÚtorchÚenable_gradr%   ÚgradÚappendÚ	is_sparseÚRuntimeErrorr'   ÚlenÚ
zeros_likeÚpreserve_formatÚlionr   )r   ÚclosureÚlossr(   Úparams_with_gradÚgradsÚexp_avgsr,   r-   Úpr'   s              r!   Ústepz	Lion.stepQ   sØ  € ð ˆØÐÝÔ"Ñ$Ô$ð !ð !Ø�w‘y”y�ð!ð !ð !ñ !ô !ð !ð !ð !ð !ð !ð !øøøð !ð !ð !ð !ð Ô&ð "	ñ "	ˆEØ!ÐØˆEØˆHØ  œ>‰LˆE�5à˜8”_ð 2ð 2�Ø”6�>ØØ ×'Ò'¨Ñ*Ô*Ð*Ø”6Ô#ð QÝ&Ð'OÑPÔPÐPØ—’˜QœVÑ$Ô$Ð$àœ
 1œ�õ �u‘:”: ’?�?Ý',Ô'7¸ÍÔI^Ð'_Ñ'_Ô'_�E˜)Ñ$à—’  iÔ 0Ñ1Ô1Ð1Ð1åØ ØØØØØ˜”;Ø" >Ô2Ø˜iÔ(Ø˜zÔ*Ø˜iÔ(Ø.3Ð4LÔ.MÐW�t”} TÔ*Ð*ÐSWðñ ô ð ñ ð ˆs   ˜/¯3¶3)r   r   r   FFFN)N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r
   Úfloatr   Úboolr   r   r$   r/   Úno_gradr?   Ú__classcell__)r    s   @r!   r   r       sü   ø€ € € € € Ø%Ð%ð
 Ø)4Ø"%Ø!Ø+0Ø"Ø&*ð$+ð $+àð$+ð ð$+ð ˜ ˜Ô&ð	$+ð
  ð$+ð ð$+ð %)ð$+ð ð$+ð ˜d”^ð$+ð $+ð $+ð $+ð $+ð $+ðL.ð .ð .ð .ð .ð €U„]�_„_ð2ð 2ð 2ñ „_ð2ð 2ð 2ð 2ð 2r"   r   F)r.   r   r<   r=   r   r   r,   r-   r   r   r   r.   c                ó¢  — |€Y	 |	 p)dt           j        j        j                             ¦   «         v }|rt          j        |¦  «        rd}n# t          $ r d}Y nw xY w|r-t           j                             ¦   «         rt          d¦  «        ‚|r&t           j                             ¦   «         st          }nt          } || |||||||	||
¬¦
  «
         dS )z=Functional API that performs Lion algorithm computation.
    NÚScalarFz6torch.jit.script not supported with foreach optimizers)r,   r-   r   r   r   r   r.   )r/   ÚopsÚatenÚ_foreach_maximum_Ú	overloadsÚ	is_tensorÚ	ExceptionÚjitÚis_scriptingr4   Ú_multi_tensor_lionÚ_single_tensor_lion)r   r<   r=   r   r   r,   r-   r   r   r   r.   Úfuncs               r!   r8   r8   ‡   s  € ð$ €ð	à!�kÐ] Xµ´´Ô1Q×1[Ò1[Ñ1]Ô1]Ð%]ˆGØð  �5œ?¨2Ñ.Ô.ð  Ø�øøÝð 	ð 	ð 	ØˆGˆGˆGð	øøøð ð U•5”9×)Ò)Ñ+Ô+ð UÝÐSÑTÔTÐTàð #•u”y×-Ò-Ñ/Ô/ð #Ý!ˆˆå"ˆà€DØØØØØØØ!ØØØðñ ô ð ð ð s   „AA
 Á
AÁAc                ó  — t          | ¦  «        D �]u\  }
}|s||
         n||
          }||
         }t          j        |¦  «        r<t          j        |¦  «        }t          j        |¦  «        }t          j        |¦  «        }|	€|n|dz  |	z  }|                     d||z  z
  ¦  «         |                     |¦  «                             |d|z
  ¬¦  «                             ¦   «         }|rq||z  dk                         |j	        ¦  «        }| 
                    |                     ¦   «                              d¬¦  «        ¦  «         |                     |¦  «         t          ||| ¦  «         |                     |d|z
  ¦  «         �Œwd S )Né   r   ©Úalphar   çü©ñÒMbP?)Úmin)Ú	enumerater/   Ú
is_complexÚview_as_realÚmul_ÚmulÚadd_Úsign_ÚtoÚdtypeÚdiv_ÚmeanÚclamp_r   Úlerp_)r   r<   r=   r,   r-   r   r   r   r   r.   ÚiÚparamr1   r+   Úwd_scaleÚupdateÚmasks                    r!   rS   rS   ¸   s‹  € õ ˜fÑ%Ô%ð 'ñ '‰ˆˆ5Ø'Ð6ˆu�QŒxˆx¨e°A¬h¨YˆØ˜1”+ˆåÔ˜EÑ"Ô"ð 	.ÝÔ% dÑ+Ô+ˆDÝÔ(¨Ñ1Ô1ˆGÝÔ& uÑ-Ô-ˆEð  ˜�2�2¨R°1©W°vÑ-=ˆØ�
Š
�1�x ,Ñ.Ñ.Ñ/Ô/Ð/ð —’˜UÑ#Ô#×(Ò(¨°Q¸±YÐ(Ñ?Ô?×EÒEÑGÔGˆàð 	à˜T‘M AÒ%×)Ò)¨$¬*Ñ5Ô5ˆDØ�IŠI�d—i’i‘k”k×(Ò(¨TÐ(Ñ2Ô2Ñ3Ô3Ð3Ø�KŠK˜ÑÔÐå�U˜F R CÑ(Ô(Ð(ð 	�Š�d˜A ™IÑ&Ô&Ð&Ñ&ð3'ð 'r"   c                ó  — t          | ¦  «        dk    rd S |r!t          j        t          |¦  «        ¦  «        }d„ |D ¦   «         }d„ |D ¦   «         }d„ | D ¦   «         } |	€|n|dz  |	z  }
t          j        | d|
|z  z
  ¦  «         t          j        ||¦  «        }t          j        ||d|z
  ¬¦  «         d„ |D ¦   «         }|rzt          j        ||¦  «        }d	„ t          ||¦  «        D ¦   «         }d
„ |D ¦   «         }t          j        |d¦  «         t          j	        ||¦  «         t          j        ||¦  «         t          j        | || ¬¦  «         t          j        ||¦  «         t          j        ||d|z
  ¬¦  «         d S )Nr   c                 ób   — g | ],}t          j        |¦  «        rt          j        |¦  «        n|‘Œ-S © ©r/   r\   r]   ©Ú.0Úxs     r!   ú
<listcomp>z&_multi_tensor_lion.<locals>.<listcomp>ô   s8   € ÐPÐPÐPÀQ¥eÔ&6°qÑ&9Ô&9Ð@�UÔ Ñ"Ô"Ð"¸qÐPÐPÐPr"   c                 ób   — g | ],}t          j        |¦  «        rt          j        |¦  «        n|‘Œ-S ro   rp   rq   s     r!   rt   z&_multi_tensor_lion.<locals>.<listcomp>õ   s8   € ÐVÐVÐVÈ­Ô)9¸!Ñ)<Ô)<ÐC•Ô" 1Ñ%Ô%Ð%À!ÐVÐVÐVr"   c                 ób   — g | ],}t          j        |¦  «        rt          j        |¦  «        n|‘Œ-S ro   rp   rq   s     r!   rt   z&_multi_tensor_lion.<locals>.<listcomp>ö   s8   € ÐRÐRÐRÀa¥uÔ'7¸Ñ':Ô':ÐA�eÔ  Ñ#Ô#Ð#ÀÐRÐRÐRr"   rV   r   rW   c                 ó6   — g | ]}|                      ¦   «         ‘ŒS ro   )ra   )rr   Úus     r!   rt   z&_multi_tensor_lion.<locals>.<listcomp>ÿ   s    € Ð*Ð*Ð*˜Qˆq�wŠw‰yŒyÐ*Ð*Ð*r"   c                 óP   — g | ]#\  }}|d k                          |j        ¦  «        ‘Œ$S )r   )rb   rc   )rr   ÚmÚgs      r!   rt   z&_multi_tensor_lion.<locals>.<listcomp>  s.   € ÐCÐCÐC©¨¨A�!�a’%—’˜AœGÑ$Ô$ÐCÐCÐCr"   c                 ó6   — g | ]}|                      ¦   «         ‘ŒS ro   )re   )rr   rz   s     r!   rt   z&_multi_tensor_lion.<locals>.<listcomp>  s    € Ð.Ð.Ð. 1�a—f’f‘h”hÐ.Ð.Ð.r"   rY   )
r5   r/   Ú_foreach_negÚtupleÚ_foreach_mul_Ú_foreach_mulÚ_foreach_add_ÚziprL   Ú_foreach_div_)r   r<   r=   r,   r-   r   r   r   r   r.   rj   ÚupdatesÚmasksÚ
mask_scales                 r!   rR   rR   á   s½  € õ ˆ6�{„{�aÒÐØˆàð 1ÝÔ"¥5¨¡<¤<Ñ0Ô0ˆàPÐPÈ%ÐPÑPÔP€EØVÐVÈXÐVÑVÔV€HØRÐRÈ6ÐRÑRÔR€Fð �^ˆrˆr¨¨q©°6Ñ)9€HÝ	Ô˜  H¨|Ñ$;Ñ ;Ñ<Ô<Ð<õ Ô  ¨5Ñ1Ô1€GÝ	Ô˜ ¨a°%©iÐ8Ñ8Ô8Ð8Ø*Ð* 'Ð*Ñ*Ô*€Gàð ,åÔ" 7¨EÑ2Ô2ˆØCÐCµ°U¸EÑ1BÔ1BÐCÑCÔCˆØ.Ð.¨Ð.Ñ.Ô.ˆ
ÝÔ 
¨DÑ1Ô1Ð1ÝÔ˜E :Ñ.Ô.Ð.ÝÔ˜G UÑ+Ô+Ð+å	Ô˜ °¨sÐ3Ñ3Ô3Ð3õ 
Ô˜ %Ñ(Ô(Ð(Ý	Ô˜ %¨q°5©yÐ9Ñ9Ô9Ð9Ð9Ð9r"   )FN)rC   Útypingr   r   r   r/   Útorch.optim.optimizerr   Ú_helpersr   r	   Ú_typesr
   r   ÚTensorrE   rD   r8   rS   rR   ro   r"   r!   ú<module>rŒ      s€  ððð ð, )Ð (Ð (Ð (Ð (Ð (Ð (Ð (Ð (Ð (à €€€Ø +Ð +Ð +Ð +Ð +Ð +à 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4Ø Ð Ð Ð Ð Ð ðdð dð dð dð dˆ9ñ dô dð dðZ Øð.ð #'ð.ð .ð .Ø�U”\Ô"ð.à�E”LÔ!ð.ð �u”|Ô$ð.ð ð.ð ð.ð ð.ð ð.ð ð.ð ð.ð ð.ð ˜”ð.ð .ð .ð .ðb&'Ø�U”\Ô"ð&'à�E”LÔ!ð&'ð �u”|Ô$ð&'ð
 ð&'ð ð&'ð ð&'ð ð&'ð ð&'ð ð&'ð ˜”ð&'ð &'ð &'ð &'ðR-:Ø�U”\Ô"ð-:à�E”LÔ!ð-:ð �u”|Ô$ð-:ð
 ð-:ð ð-:ð ð-:ð ð-:ð ð-:ð ð-:ð ˜”ð-:ð -:ð -:ð -:ð -:ð -:r"   