§
    ·¯¸j2'  ã                   ó>   — d dl Z d dlZd dlmZ  G d„ de¦  «        ZdS )é    N)Ú	Optimizerc                   ó¢   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 dˆ fd„	Zˆ fd	„Z ej        ¦   «         d
„ ¦   «         Z ej        ¦   «         dd„¦   «         Z	ˆ xZ
S )Ú	AdaBeliefa‹  Implements AdaBelief algorithm. Modified from Adam in PyTorch

    Arguments:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-3)
        betas (Tuple[float, float], optional): coefficients used for computing
            running averages of gradient and its square (default: (0.9, 0.999))
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-16)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        amsgrad (boolean, optional): whether to use the AMSGrad variant of this
            algorithm from the paper `On the Convergence of Adam and Beyond`_
            (default: False)
        decoupled_decay (boolean, optional): (default: True) If set as True, then
            the optimizer uses decoupled weight decay as in AdamW
        fixed_decay (boolean, optional): (default: False) This is used when weight_decouple
            is set as True.
            When fixed_decay == True, the weight decay is performed as
            $W_{new} = W_{old} - W_{old} \times decay$.
            When fixed_decay == False, the weight decay is performed as
            $W_{new} = W_{old} - W_{old} \times decay \times lr$. Note that in this case, the
            weight decay ratio decreases with learning rate (lr).
        rectify (boolean, optional): (default: True) If set as True, then perform the rectified
            update similar to RAdam
        degenerated_to_sgd (boolean, optional) (default:True) If set as True, then perform SGD update
            when variance of gradient is high
    reference: AdaBelief Optimizer, adapting stepsizes by the belief in observed gradients, NeurIPS 2020

    For a complete table of recommended hyperparameters, see https://github.com/juntang-zhuang/Adabelief-Optimizer'
    For example train/args for EfficientNet see these gists
      - link to train_script: https://gist.github.com/juntang-zhuang/0a501dd51c02278d952cf159bc233037
      - link to args.yaml: https://gist.github.com/juntang-zhuang/517ce3c27022b908bb93f78e4f786dc3
    çü©ñÒMbP?©gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?ç¼‰Ø—²Òœ<r   FTc                 óŒ  •— d|k    s"t          d                     |¦  «        ¦  «        ‚d|k    s"t          d                     |¦  «        ¦  «        ‚d|d         cxk    rdk     s*n t          d                     |d         ¦  «        ¦  «        ‚d|d         cxk    rdk     s*n t          d                     |d         ¦  «        ¦  «        ‚t          |t          t          f¦  «        rƒt          |¦  «        dk    rpt          |d         t          ¦  «        rU|D ]R}d	|v rL|d	         d         |d         k    s|d	         d         |d         k    rd
„ t          d¦  «        D ¦   «         |d<   ŒSt          ||||||
||	|d„ t          d¦  «        D ¦   «         ¬¦
  «
        }t          t          | ¦  «         
                    ||¦  «         d S )Ng        zInvalid learning rate: {}zInvalid epsilon value: {}r   ç      ð?z%Invalid beta parameter at index 0: {}é   z%Invalid beta parameter at index 1: {}Úbetasc                 ó   — g | ]}g d ¢‘ŒS ©)NNN© ©Ú.0Ú_s     úd/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/optim/adabelief.pyú
<listcomp>z&AdaBelief.__init__.<locals>.<listcomp>C   s   € Ð&MÐ&MÐ&M¸aÐ'9Ð'9Ð'9Ð&MÐ&MÐ&Mó    é
   Úbufferc                 ó   — g | ]}g d ¢‘ŒS r   r   r   s     r   r   z&AdaBelief.__init__.<locals>.<listcomp>O   s   € Ð:Ð:Ð:¨1Ð&Ð&Ð&Ð:Ð:Ð:r   )
Úlrr   ÚepsÚweight_decayÚamsgradÚdegenerated_to_sgdÚdecoupled_decayÚrectifyÚfixed_decayr   )Ú
ValueErrorÚformatÚ
isinstanceÚlistÚtupleÚlenÚdictÚrangeÚsuperr   Ú__init__)ÚselfÚparamsr   r   r   r   r   r   r    r   r   ÚparamÚdefaultsÚ	__class__s                €r   r*   zAdaBelief.__init__*   sñ  ø€ ð �bŠyˆyÝÐ8×?Ò?ÀÑCÔCÑDÔDÐDØ�cŠzˆzÝÐ8×?Ò?ÀÑDÔDÑEÔEÐEØ�e˜A”hÐ$Ð$Ò$Ð$ Ò$Ð$Ð$Ð$ÝÐD×KÒKÈEÐRSÌHÑUÔUÑVÔVÐVØ�e˜A”hÐ$Ð$Ò$Ð$ Ò$Ð$Ð$Ð$ÝÐD×KÒKÈEÐRSÌHÑUÔUÑVÔVÐVå�f�t¥U˜mÑ,Ô,ð 	Nµ°V±´¸q²°ÅZÐPVÐWXÔPYÕ[_ÑE`ÔE`°Øð Nð N�Ø˜eÐ#Ð#¨¨w¬¸Ô):¸eÀA¼hÒ)FÐ)FÈ%ÐPWÌ.ÐYZÔJ[Ð_dÐefÔ_gÒJgÐJgØ&MÐ&MÅ5ÈÁ9Ä9Ð&MÑ&MÔ&M�E˜(‘OøåØØØØ%ØØ1Ø+ØØ#Ø:Ð:µ°b±	´	Ð:Ñ:Ô:ð
ñ 
ô 
ˆõ 	�i˜ÑÔ×'Ò'¨°Ñ9Ô9Ð9Ð9Ð9r   c                 ó˜   •— t          t          | ¦  «                             |¦  «         | j        D ]}|                     dd¦  «         Œd S )Nr   F)r)   r   Ú__setstate__Úparam_groupsÚ
setdefault)r+   ÚstateÚgroupr/   s      €r   r1   zAdaBelief.__setstate__S   sV   ø€ Ý�i˜ÑÔ×+Ò+¨EÑ2Ô2Ð2ØÔ&ð 	/ð 	/ˆEØ×Ò˜Y¨Ñ.Ô.Ð.Ð.ð	/ð 	/r   c                 óò   — | j         D ]n}|d         D ]c}| j        |         }|d         }d|d<   t          j        |¦  «        |d<   t          j        |¦  «        |d<   |rt          j        |¦  «        |d<   ŒdŒod S )Nr,   r   r   ÚstepÚexp_avgÚexp_avg_varÚmax_exp_avg_var)r2   r4   ÚtorchÚ
zeros_like)r+   r5   Úpr4   r   s        r   ÚresetzAdaBelief.resetX   s©   € àÔ&ð 	Cð 	CˆEØ˜8”_ð Cð C�Øœ
 1œ�Ø 	Ô*�ð !"��f‘å#(Ô#3°AÑ#6Ô#6��iÑ õ (-Ô'7¸Ñ':Ô':��mÑ$Øð Cå/4Ô/?ÀÑ/BÔ/B�EÐ+Ñ,øðCð	Cð 	Cr   Nc           	      óH
  — d}|�5t          j        ¦   «         5   |¦   «         }ddd¦  «         n# 1 swxY w Y   | j        D �]ß}|d         D �]Ò}|j        €Œ|j        }|j        t           j        t           j        hv r|                     ¦   «         }|j        rt          d¦  «        ‚|}|j        t           j        t           j        hv r|                     ¦   «         }|d         }|d         \  }}	| j
        |         }
t          |
¦  «        dk    rLd|
d<   t          j        |¦  «        |
d<   t          j        |¦  «        |
d	<   |rt          j        |¦  «        |
d
<   |d         rO|d         s(|                     d|d         |d         z  z
  ¦  «         nH|                     d|d         z
  ¦  «         n)|d         dk    r|                     ||d         ¬¦  «         |
d         |
d	         }}|
dxx         dz  cc<   d||
d         z  z
  }d|	|
d         z  z
  }|                     |¦  «                             |d|z
  ¬¦  «         ||z
  }|                     |	¦  «                             ||d|	z
  ¬¦  «         |r{|
d
         }t          j        ||                     |d         ¦  «        |¬¦  «         |                     ¦   «         t%          j        |¦  «        z                       |d         ¦  «        }n[|                     |d         ¦  «                             ¦   «         t%          j        |¦  «        z                       |d         ¦  «        }|d         s&|d         |z  }|                     ||| ¬¦  «         �nm|d         t)          |
d         dz  ¦  «                 }|
d         |d         k    r|d         |d         }}n©|
d         |d<   |	|
d         z  }dd|	z
  z  dz
  }|d|
d         z  |z  d|z
  z  z
  }||d<   |dk    rEt%          j        d|z
  |dz
  z  |dz
  z  |dz
  z  |z  |z  |dz
  z  ¦  «        d||
d         z  z
  z  }n|d         rdd||
d         z  z
  z  }nd}||d<   |dk    rP|                     ¦   «                              |d         ¦  «        }|                     ||| |d         z  ¬¦  «         n'|dk    r!|                     || |d         z  ¬¦  «         |j        t           j        t           j        hv r|                     |¦  «         �ŒÔ�Œá|S )zµPerforms a single optimization step.
        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr,   zOAdaBelief does not support sparse gradients, please consider SparseAdam insteadr   r   r   r7   r8   r9   r:   r   r    r
   r   r   )Úalphar   )Úvaluer   )Úoutr   r   r   é   é   é   r   éÿÿÿÿ)r;   Úenable_gradr2   ÚgradÚdtypeÚfloat16Úbfloat16ÚfloatÚ	is_sparseÚRuntimeErrorr4   r&   r<   Úmul_Úadd_Úaddcmul_ÚmaxÚsqrtÚmathÚaddcdiv_ÚintÚcopy_)r+   ÚclosureÚlossr5   r=   rH   Úp_fp32r   Úbeta1Úbeta2r4   r8   r9   Úbias_correction1Úbias_correction2Úgrad_residualr:   ÚdenomÚ	step_sizeÚbufferedÚnum_smaÚbeta2_tÚnum_sma_maxs                          r   r7   zAdaBelief.stepj   s  € ð ˆØÐÝÔ"Ñ$Ô$ð !ð !Ø�w‘y”y�ð!ð !ð !ñ !ô !ð !ð !ð !ð !ð !ð !øøøð !ð !ð !ð !ð Ô&ð b	$ñ b	$ˆEØ˜8”_ð a$ñ a$�Ø”6�>ØØ”v�Ø”:¥%¤-µ´Ð!@Ð@Ð@ØŸ:š:™<œ<�DØ”>ð kÝ&Øiñkô kð kð �Ø”7�uœ}­e¬nÐ=Ð=Ð=Ø#Ÿ\š\™^œ^�Fà 	Ô*�Ø$ Wœ~‘��uØœ
 1œ�å�u‘:”: ’?�?Ø$%�E˜&‘Må',Ô'7¸Ñ'?Ô'?�E˜)Ñ$å+0Ô+;¸FÑ+CÔ+C�E˜-Ñ(Øð Lå38Ô3CÀFÑ3KÔ3K˜Ð/Ñ0ð Ð*Ô+ð GØ  Ô/ð AØŸš C¨%°¬+¸¸nÔ8MÑ*MÑ$MÑNÔNÐNÐNàŸš C¨%°Ô*?Ñ$?Ñ@Ô@Ð@Ð@à˜^Ô,°Ò1Ð1ØŸ	š	 &°°nÔ0E˜	ÑFÔFÐFð (-¨YÔ'7¸¸}Ô9M˜�à�f��” Ñ"��‘Ø#$ u°°f´Ñ'=Ñ#=Ð Ø#$ u°°f´Ñ'=Ñ#=Ð ð —’˜UÑ#Ô#×(Ò(¨°Q¸±YÐ(Ñ?Ô?Ð?Ø $ w¡�Ø× Ò  Ñ'Ô'×0Ò0°ÀÐUVÐY^ÑU^Ð0Ñ_Ô_Ð_àð uØ&+Ð,=Ô&>�Oå”I˜o¨{×/?Ò/?ÀÀeÄÑ/MÔ/MÐSbÐcÑcÔcÐcð -×1Ò1Ñ3Ô3µd´iÐ@PÑ6QÔ6QÑQ×WÒWÐX]Ð^cÔXdÑeÔe�E�Eà(×-Ò-¨e°E¬lÑ;Ô;×@Ò@ÑBÔBÅTÄYÐO_ÑE`ÔE`Ñ`×fÒfÐglÐmrÔgsÑtÔt�Eð ˜YÔ'ð !Mà % d¤Ð.>Ñ >�IØ—O’O G¨U¸9¸*�OÑEÔEÐEÑEð  % Xœ­s°5¸´=À2Ñ3EÑ/FÔ/FÔG�HØ˜V”}¨°¬Ò3Ð3Ø-5°a¬[¸(À1¼+ ˜˜à&+¨F¤m˜ ™Ø"'¨5°¬=Ñ"8˜Ø&'¨1¨u©9¡o¸Ñ&9˜Ø"-°°E¸&´MÑ0AÀGÑ0KÈqÐSZÉ{Ñ0[Ñ"[˜Ø&-˜ ™ð # aš<˜<Ý(,¬	Ø!" W¡Ø!(¨1¡ñ!.Ø1<¸q±ñ!Bà!(¨1¡ñ!.à07ñ!8ð !,ñ!,ð 0;¸Q©ñ!@ñ)Aô )Að EFÈÐQVÐW]ÔQ^ÑH^ÑD^ñ	)`˜I˜Ið
 #Ð#7Ô8ð +Ø(+¨q°5¸EÀ&¼MÑ3IÑ/IÑ(J˜I˜Ià(*˜IØ&/˜ ™à !’|�|Ø +× 0Ò 0Ñ 2Ô 2× 7Ò 7¸¸e¼Ñ EÔ E˜ØŸš¨°¸y¸jÈ5ÐQUÌ;Ñ>V˜ÑWÔWÐWÐWØ" Qš˜ØŸš G°I°:ÀÀdÄÑ3K˜ÑLÔLÐLà”7�uœ}­e¬nÐ=Ð=Ð=Ø—G’G˜F‘O”O�OùñCa$ðF ˆs   ˜/¯3¶3)	r   r   r   r   FTFTT)N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r*   r1   r;   Úno_gradr>   r7   Ú__classcell__)r/   s   @r   r   r      sÕ   ø€ € € € € ð!ð !ðL ØØØØØ ØØØ#ð':ð ':ð ':ð ':ð ':ð ':ðR/ð /ð /ð /ð /ð
 €U„]�_„_ðCð Cñ „_ðCð" €U„]�_„_ðoð oð oñ „_ðoð oð oð oð or   r   )rT   r;   Útorch.optim.optimizerr   r   r   r   r   ú<module>rm      sg   ðØ €€€Ø €€€Ø +Ð +Ð +Ð +Ð +Ð +ðTð Tð Tð Tð T�	ñ Tô Tð Tð Tð Tr   