§
    ·¯¸j$D  ã            %       óh  — d Z ddlmZmZmZ ddlZddlmZ ddlmZ ddl	m
Z
mZmZ ddlmZ  G d	„ d
e¦  «        Z	 	 d!dee         dee         dee         dee         dee         dee         dee         dedededededededededee         ddf$d„Zdee         dee         dee         dee         dee         dee         dedededededededededee         f d„Zdee         dee         dee         dee         dee         dee         dedededededededededee         f d „ZdS )"aM   AdamW Optimizer
Impl copied from PyTorch master

References for added functionality:
    Cautious Optimizers: https://arxiv.org/abs/2411.16085
    Why Gradients Rapidly Increase Near the End of Training: https://arxiv.org/abs/2506.02285

NOTE: This impl has been deprecated in favour of torch.optim.AdamW and remains as a reference
é    )ÚListÚOptionalÚTupleN)ÚTensor)Ú	Optimizeré   )Ú_check_capturable_devicesÚ
_get_valueÚ_init_scalar)ÚParamsTc                   ó¾   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 dded	ed
eeef         dededededededee         defˆ fd„Z	ˆ fd„Z
 ej        ¦   «         dd„¦   «         Zˆ xZS )ÚAdamWLegacya  Implements AdamW algorithm.

    NOTE: This impl has been deprecated in favour of torch.optim.AdamW and remains as a reference

    References:
        - Adam: A Method for Stochastic Optimization: https://arxiv.org/abs/1412.6980
        - Decoupled Weight Decay Regularization: https://arxiv.org/abs/1711.05101
        - On the Convergence of Adam and Beyond: https://openreview.net/forum?id=ryQu7f-RZ

    Args:
        params: iterable of parameters to optimize or dicts defining parameter groups
        lr: learning rate
        betas: coefficients used for computing running averages of gradient and its square
        eps: term added to the denominator to improve numerical stability
        weight_decay: weight decay coefficient
        amsgrad: whether to use the AMSGrad variant of this algorithm
            from the paper `On the Convergence of Adam and Beyond`
        caution: apply caution when using AdamW
        corrected_weight_decay: apply corrected weight decay (lr**2 / max_lr)
        maximize: maximize the params based on the objective, instead of minimizing
        foreach: whether foreach implementation of optimizer is used.
            If unspecified by the user (so foreach is None), we will try to use
            foreach over for-loop implementation on CUDA, since it is faster in general.
        capturable: whether this instance is safe to capture in a CUDA graph.
            Passing True can impair ungraphed performance, so if you don't intend to
            graph capture this instance, leave it False
    çü©ñÒMbP?©gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?ç:Œ0âŽyE>ç{®Gáz„?FNÚparamsÚlrÚbetasÚepsÚweight_decayÚamsgradÚcautionÚcorrected_weight_decayÚmaximizeÚforeachÚ
capturablec                 ó   •— d|k    s"t          d                     |¦  «        ¦  «        ‚d|k    s"t          d                     |¦  «        ¦  «        ‚d|d         cxk    rdk     s*n t          d                     |d         ¦  «        ¦  «        ‚d|d         cxk    rdk     s*n t          d                     |d         ¦  «        ¦  «        ‚t          ||||||||
|	|¬	¦
  «
        }t          t          | ¦  «                             ||¦  «         d S )
Ng        zInvalid learning rate: {}zInvalid epsilon value: {}r   ç      ð?z%Invalid beta parameter at index 0: {}r   z%Invalid beta parameter at index 1: {})
r   r   r   r   r   r   r   r   r   r   )Ú
ValueErrorÚformatÚdictÚsuperr   Ú__init__)Úselfr   r   r   r   r   r   r   r   r   r   r   ÚdefaultsÚ	__class__s                €ú`/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/optim/adamw.pyr$   zAdamWLegacy.__init__1   s5  ø€ ð �bŠyˆyÝÐ8×?Ò?ÀÑCÔCÑDÔDÐDØ�cŠzˆzÝÐ8×?Ò?ÀÑDÔDÑEÔEÐEØ�e˜A”hÐ$Ð$Ò$Ð$ Ò$Ð$Ð$Ð$ÝÐD×KÒKÈEÐRSÌHÑUÔUÑVÔVÐVØ�e˜A”hÐ$Ð$Ò$Ð$ Ò$Ð$Ð$Ð$ÝÐD×KÒKÈEÐRSÌHÑUÔUÑVÔVÐVÝØØØØ%ØØØ#9ØØØ!ð
ñ 
ô 
ˆõ 	�k˜4Ñ Ô ×)Ò)¨&°(Ñ;Ô;Ð;Ð;Ð;ó    c                 ó  •— t          t          | ¦  «                             |¦  «         | j        D ]Û}|                     dd¦  «         |                     dd¦  «         |                     dd¦  «         |                     dd ¦  «         |                     dd¦  «         |                     dd¦  «         |d         D ]L}| j                             |i ¦  «        }|r-d	|v r)t          |d	         |d         r|j        nd
¬¦  «        |d	<   ŒMŒÜd S )Nr   Fr   r   r   r   r   r   ÚstepÚcpu©Údevice)	r#   r   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetr   r.   )r%   r2   ÚgroupÚpÚp_stater'   s        €r(   r/   zAdamWLegacy.__setstate__U   s4  ø€ Ý�k˜4Ñ Ô ×-Ò-¨eÑ4Ô4Ð4ØÔ&ð 	ð 	ˆEØ×Ò˜Y¨Ñ.Ô.Ð.Ø×Ò˜Y¨Ñ.Ô.Ð.Ø×ÒÐ5°uÑ=Ô=Ð=Ø×Ò˜Y¨Ñ-Ô-Ð-Ø×Ò˜Z¨Ñ/Ô/Ð/Ø×Ò˜\¨5Ñ1Ô1Ð1Ø˜8”_ð ð �Øœ*Ÿ.š.¨¨BÑ/Ô/�Øð ˜v¨Ð0Ð0Ý&2Ø œØ+0°Ô+>ÐI˜qœx˜xÀEð'ñ 'ô '�G˜F‘Oøðð	ð 	r)   c                 ó.  — t          | d¦  «        r|                      ¦   «          n$t          | d¦  «        r|                      ¦   «          d}|�5t          j        ¦   «         5   |¦   «         }ddd¦  «         n# 1 swxY w Y   | j        D �]	}g }g }g }g }g }g }	|d         \  }
}|d         }|d         D �]~}|j        €Œ|                     |¦  «         |j        j        rt          d¦  «        ‚|                     |j        ¦  «         | j
        |         }t          |¦  «        dk    r�t          |d	         r|j        nd
¬¦  «        |d<   t          j        |t          j        ¬¦  «        |d<   t          j        |t          j        ¬¦  «        |d<   |r#t          j        |t          j        ¬¦  «        |d<   |                     |d         ¦  «         |                     |d         ¦  «         |r)|                     |                     dd¦  «        ¦  «         |	                     |d         ¦  «         �Œ€t#          ||||||	|d         ||
||d         |d         |d         |d         |d         |d	         |d         r| j        d         nd¬¦  «         �Œ|S )z¶Performs a single optimization step.

        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        Ú'_accelerator_graph_capture_health_checkÚ _cuda_graph_capture_health_checkNr   r   r   z'AdamW does not support sparse gradientsr   r   r,   r-   r+   )Úmemory_formatÚexp_avgÚ
exp_avg_sqÚmax_exp_avg_sqr   r   r   r   r   r   r   )r   r   Úbeta1Úbeta2r   r   r   r   r   r   Úmax_lr)Úhasattrr8   r9   ÚtorchÚenable_gradr0   ÚgradÚappendÚ	is_sparseÚRuntimeErrorr2   Úlenr   r.   Ú
zeros_likeÚpreserve_formatr3   Úadamwr&   )r%   ÚclosureÚlossr4   Úparams_with_gradÚgradsÚexp_avgsÚexp_avg_sqsÚmax_exp_avg_sqsÚstate_stepsr>   r?   r   r5   r2   s                  r(   r+   zAdamWLegacy.stepf   s   € õ �4ÐBÑCÔCð 	4Ø×8Ò8Ñ:Ô:Ð:Ð:Ý�TÐ=Ñ>Ô>ð 	4Ø×1Ò1Ñ3Ô3Ð3àˆØÐÝÔ"Ñ$Ô$ð !ð !Ø�w‘y”y�ð!ð !ð !ñ !ô !ð !ð !ð !ð !ð !ð !øøøð !ð !ð !ð !ð Ô&ð 7	ñ 7	ˆEØ!ÐØˆEØˆHØˆKØ ˆOØˆKØ  œ>‰LˆE�5Ø˜IÔ&ˆGà˜8”_ð 2ñ 2�Ø”6�>ØØ ×'Ò'¨Ñ*Ô*Ð*Ø”6Ô#ð RÝ&Ð'PÑQÔQÐQØ—’˜QœVÑ$Ô$Ð$àœ
 1œ�õ �u‘:”: ’?�?Ý$0ÀEÈ,ÔDWÐ8b¸¼¸Ð]bÐ$cÑ$cÔ$c�E˜&‘Må',Ô'7¸ÍÔI^Ð'_Ñ'_Ô'_�E˜)Ñ$å*/Ô*:¸1ÍEÔLaÐ*bÑ*bÔ*b�E˜,Ñ'Øð kå27Ô2BÀ1ÕTYÔTiÐ2jÑ2jÔ2j˜Ð.Ñ/à—’  iÔ 0Ñ1Ô1Ð1Ø×"Ò" 5¨Ô#6Ñ7Ô7Ð7Øð NØ#×*Ò*¨5¯9ª9Ð5EÀtÑ+LÔ+LÑMÔMÐMØ×"Ò" 5¨¤=Ñ1Ô1Ð1Ñ1åØ ØØØØØØ˜iÔ(ØØØØ˜”;Ø" >Ô2Ø˜%”LØ˜iÔ(Ø˜zÔ*Ø  Ô.Ø.3Ð4LÔ.MÐW�t”} TÔ*Ð*ÐSWð#ñ ô ð ñ ð( ˆs   Á!A8Á8A<Á?A<)
r   r   r   r   FFFFNF©N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Úfloatr   Úboolr   r$   r/   rB   Úno_gradr+   Ú__classcell__)r'   s   @r(   r   r      s/  ø€ € € € € ðð ð> Ø)5ØØ"&Ø!Ø!Ø+0Ø"Ø&*Ø$ð"<ð "<àð"<ð ð"<ð ˜ ˜Ô&ð	"<ð
 ð"<ð  ð"<ð ð"<ð ð"<ð %)ð"<ð ð"<ð ˜d”^ð"<ð ð"<ð "<ð "<ð "<ð "<ð "<ðHð ð ð ð ð" €U„]�_„_ðJð Jð Jñ „_ðJð Jð Jð Jð Jr)   r   Fr   rO   rP   rQ   rR   rS   r   r   r   r>   r?   r   r   r   r   r   r@   Úreturnc       	         ó¤  — t          d„ |D ¦   «         ¦  «        st          d¦  «        ‚|€[	 | p)dt          j        j        j                             ¦   «         v }|rt          j        |¦  «        r|sd}n# t          $ r d}Y nw xY w|r&t          j	         
                    ¦   «         st          }nt          } || |||||||	|
|||||||¬¦  «         dS )zgFunctional API that performs AdamW algorithm computation.
      See AdamWLegacy class for details.
    c              3   óJ   K  — | ]}t          |t          j        ¦  «        V — Œd S rT   )Ú
isinstancerB   r   )Ú.0Úts     r(   ú	<genexpr>zadamw.<locals>.<genexpr>Ì   s.   è è € Ð@Ð@¨q�z˜!�Uœ\Ñ*Ô*Ð@Ð@Ð@Ð@Ð@Ð@r)   zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsNÚScalarF)
r   r>   r?   r   r   r   r   r   r   r@   )ÚallrG   rB   ÚopsÚatenÚ_foreach_maximum_Ú	overloadsÚ	is_tensorÚ	ExceptionÚjitÚis_scriptingÚ_multi_tensor_adamwÚ_single_tensor_adamw)r   rO   rP   rQ   rR   rS   r   r   r   r>   r?   r   r   r   r   r   r@   Úfuncs                     r(   rK   rK   ´   s2  € õ0 Ð@Ð@°KÐ@Ñ@Ô@Ñ@Ô@ð "Ýð!ñ"ô "ð 	"ð €ð	à!�kÐ] Xµ´´Ô1Q×1[Ò1[Ñ1]Ô1]Ð%]ˆGàð  �5œ?¨2Ñ.Ô.ð  °zð  Ø�øøÝð 	ð 	ð 	ØˆGˆGˆGð	øøøð ð $•u”y×-Ò-Ñ/Ô/ð $Ý"ˆˆå#ˆà€DØØØØØØØØØØØ!ØØØØØð!ñ ô ð ð ð s   ¬AA4 Á4BÂBc       
         óÀ  — |rt          | |¦  «         t          | ¦  «        D �]º\  }}|s||         n||          }||         }||         }||         }|dz  }|€|	n|	dz  |z  }|                     d||
z  z
  ¦  «         |                     |¦  «                             |d|z
  ¬¦  «         |                     |¦  «                             ||d|z
  ¬¦  «         |r"||         }t          j        |||¬¦  «         |}n|}|�r
|}dt          j        ||¦  «        z
  }dt          j        ||¦  «        z
  }|	|z  }|                     ¦   «         }| 	                    ¦   «         }| 	                    ¦   «         ||z  z                       ||z  ¦  «        }|ra||z  dk     
                    |j        ¦  «        } |                      |                      ¦   «                              d¬	¦  «        ¦  «         || z  }|                     ||¦  «         �Œët!          |¦  «        }d||z  z
  }d||z  z
  }|	|z  }|d
z  }| 	                    ¦   «         |z                       |¦  «        }|ra||z  dk     
                    |j        ¦  «        } |                      |                      ¦   «                              d¬	¦  «        ¦  «         || z  }|                     ||| ¬¦  «         �Œ¼d S )Nr   é   r   ©Úalpha)Úvalue)Úoutr   r   )Úminç      à?)r	   Ú	enumerateÚmul_Úadd_Úaddcmul_rB   ÚmaxÚpowÚnegÚsqrtÚtoÚdtypeÚdiv_ÚmeanÚclamp_Úaddcdiv_r
   )!r   rO   rP   rQ   rR   rS   r   r>   r?   r   r   r   r   r   r   r@   ÚiÚparamrD   r;   r<   Ústep_tÚwd_scaler=   Ú
denom_baser+   Úbias_correction1Úbias_correction2Ú	step_sizeÚstep_size_negÚbias_correction2_sqrtÚdenomÚmasks!                                    r(   ro   ro   ô   s#  € ð& ð 7Ý! &¨+Ñ6Ô6Ð6å˜fÑ%Ô%ð ?=ñ ?=‰ˆˆ5Ø'Ð6ˆu�QŒxˆx¨e°A¬h¨YˆØ˜1”+ˆØ  ”^ˆ
Ø˜Q”ˆð 	�!‰ˆð  ˜�2�2¨R°1©W°vÑ-=ˆØ�
Š
�2˜ <Ñ/Ñ/Ñ0Ô0Ð0ð 	�Š�UÑÔ× Ò  ¨Q°©YÐ Ñ7Ô7Ð7Ø�Š˜ÑÔ×'Ò'¨¨d¸!¸e¹)Ð'ÑDÔDÐDàð 	$Ø,¨QÔ/ˆNåŒI�n j°nÐEÑEÔEÐEØ'ˆJˆJà#ˆJàñ &	=ØˆDð  !¥5¤9¨U°DÑ#9Ô#9Ñ9ÐØ ¥5¤9¨U°DÑ#9Ô#9Ñ9ÐàÐ-Ñ-ˆIØ%ŸMšM™OœOˆMà$4×$9Ò$9Ñ$;Ô$;Ð!à—_’_Ñ&Ô&Ð*?À-Ñ*OÑP×VÒVÐWZÐ]jÑWjÑkÔkˆEàð )ð   $™¨Ò*×.Ò.¨t¬zÑ:Ô:�Ø—	’	˜$Ÿ)š)™+œ+×,Ò,°Ð,Ñ6Ô6Ñ7Ô7Ð7Ø! D™.�à�NŠN˜7 EÑ*Ô*Ð*Ñ*å˜fÑ%Ô%ˆDØ  5¨D¡=Ñ0ÐØ  5¨D¡=Ñ0ÐØÐ-Ñ-ˆIØ$4¸Ñ$;Ð!à—_’_Ñ&Ô&Ð)>Ñ>×DÒDÀSÑIÔIˆEàð )à $™¨Ò*×.Ò.¨t¬zÑ:Ô:�Ø—	’	˜$Ÿ)š)™+œ+×,Ò,°Ð,Ñ6Ô6Ñ7Ô7Ð7Ø! D™.�à�NŠN˜7 E°)°ˆNÑ<Ô<Ð<Ñ<ð?=ð ?=r)   c       
         óê  ‡‡‡	— t          | ¦  «        dk    rd S |rt          | |¦  «         |r!t          j        t	          |¦  «        ¦  «        }d„ |D ¦   «         }d„ |D ¦   «         }d„ |D ¦   «         }d„ | D ¦   «         } t          j        |d¦  «         |€‰	n‰	dz  |z  }t          j        | d||
z  z
  ¦  «         t          j        |‰¦  «         t          j        ||d‰z
  ¬¦  «         t          j        |‰¦  «         t          j        |||d‰z
  ¦  «         |�rˆfd	„|D ¦   «         }ˆfd
„|D ¦   «         }t          j        |d¦  «         t          j        |d¦  «         t          j	        |¦  «         t          j	        |¦  «         t          j
        |‰	¦  «        }t          j        |¦  «         t          j	        |¦  «         t          j        |¦  «        }|r6d„ |D ¦   «         }t          j        ||¦  «         t          j        |¦  «        }nt          j        |¦  «        }t          j        |t          j        ||¦  «        ¦  «         t          j
        ||¦  «        }t          j        |¦  «         t          j        ||¦  «        }|rzt          j        ||¦  «        }d„ t#          ||¦  «        D ¦   «         }d„ |D ¦   «         }t          j        |d¦  «         t          j        ||¦  «         t          j        ||¦  «        }t          j        | ||¦  «         d S ˆfd„|D ¦   «         }ˆfd„|D ¦   «         }ˆ	fd„|D ¦   «         }d„ |D ¦   «         }|r6d„ |D ¦   «         }t          j        ||¦  «         t          j        |¦  «        }nt          j        |¦  «        }t          j        ||¦  «         t          j        ||¦  «         |rzt          j        ||¦  «        }d„ t#          ||¦  «        D ¦   «         }d„ |D ¦   «         }t          j        |d¦  «         t          j        ||¦  «         t          j        ||¦  «        }t          j        ||¦  «         t          j        | ||¦  «         d S )Nr   c                 ób   — g | ],}t          j        |¦  «        rt          j        |¦  «        n|‘Œ-S © ©rB   Ú
is_complexÚview_as_real©ra   Úxs     r(   ú
<listcomp>z'_multi_tensor_adamw.<locals>.<listcomp>h  s8   € ÐPÐPÐPÀQ¥eÔ&6°qÑ&9Ô&9Ð@�UÔ Ñ"Ô"Ð"¸qÐPÐPÐPr)   c                 ób   — g | ],}t          j        |¦  «        rt          j        |¦  «        n|‘Œ-S r•   r–   r™   s     r(   r›   z'_multi_tensor_adamw.<locals>.<listcomp>i  s8   € ÐVÐVÐVÈ­Ô)9¸!Ñ)<Ô)<ÐC•Ô" 1Ñ%Ô%Ð%À!ÐVÐVÐVr)   c                 ób   — g | ],}t          j        |¦  «        rt          j        |¦  «        n|‘Œ-S r•   r–   r™   s     r(   r›   z'_multi_tensor_adamw.<locals>.<listcomp>j  s8   € Ð\Ð\Ð\È1­EÔ,<¸QÑ,?Ô,?ÐF•5Ô% aÑ(Ô(Ð(ÀQÐ\Ð\Ð\r)   c                 ób   — g | ],}t          j        |¦  «        rt          j        |¦  «        n|‘Œ-S r•   r–   r™   s     r(   r›   z'_multi_tensor_adamw.<locals>.<listcomp>k  s8   € ÐRÐRÐRÀa¥uÔ'7¸Ñ':Ô':ÐA�eÔ  Ñ#Ô#Ð#ÀÐRÐRÐRr)   r   rr   rs   c                 ó:   •— g | ]}t          j        ‰|¦  «        ‘ŒS r•   ©rB   r~   ©ra   r+   r>   s     €r(   r›   z'_multi_tensor_adamw.<locals>.<listcomp>~  ó%   ø€ ÐKÐKÐK°t�EœI e¨TÑ2Ô2ÐKÐKÐKr)   c                 ó:   •— g | ]}t          j        ‰|¦  «        ‘ŒS r•   r    ©ra   r+   r?   s     €r(   r›   z'_multi_tensor_adamw.<locals>.<listcomp>  r¢   r)   c                 ób   — g | ],}t          j        |¦  «        rt          j        |¦  «        n|‘Œ-S r•   r–   r™   s     r(   r›   z'_multi_tensor_adamw.<locals>.<listcomp>�  ó:   € ÐlÐlÐlÐWX½Ô8HÈÑ8KÔ8KÐR�uÔ1°!Ñ4Ô4Ð4ÐQRÐlÐlÐlr)   c                 óP   — g | ]#\  }}|d k                          |j        ¦  «        ‘Œ$S ©r   ©r�   r‚   ©ra   ÚmÚgs      r(   r›   z'_multi_tensor_adamw.<locals>.<listcomp>   ó.   € ÐGÐGÐG©T¨Q°�a˜!’e—Z’Z ¤Ñ(Ô(ÐGÐGÐGr)   c                 ó6   — g | ]}|                      ¦   «         ‘ŒS r•   ©r„   ©ra   r«   s     r(   r›   z'_multi_tensor_adamw.<locals>.<listcomp>¡  ó    € Ð2Ð2Ð2 q˜!Ÿ&š&™(œ(Ð2Ð2Ð2r)   r   c                 ó:   •— g | ]}d ‰t          |¦  «        z  z
  ‘ŒS ©r   ©r
   r¡   s     €r(   r›   z'_multi_tensor_adamw.<locals>.<listcomp>©  ó+   ø€ ÐRÐRÐR¸d˜A ­°DÑ)9Ô)9Ñ 9Ñ9ÐRÐRÐRr)   c                 ó:   •— g | ]}d ‰t          |¦  «        z  z
  ‘ŒS r³   r´   r¤   s     €r(   r›   z'_multi_tensor_adamw.<locals>.<listcomp>ª  rµ   r)   c                 ó    •— g | ]
}‰|z  d z  ‘ŒS )éÿÿÿÿr•   )ra   Úbcr   s     €r(   r›   z'_multi_tensor_adamw.<locals>.<listcomp>¬  s!   ø€ Ð?Ð?Ð?¨�b˜2‘g ‘^Ð?Ð?Ð?r)   c                 ó   — g | ]}|d z  ‘ŒS )rx   r•   )ra   r¹   s     r(   r›   z'_multi_tensor_adamw.<locals>.<listcomp>®  s   € Ð FÐ FÐ F¨r  s¡Ð FÐ FÐ Fr)   c                 ób   — g | ],}t          j        |¦  «        rt          j        |¦  «        n|‘Œ-S r•   r–   r™   s     r(   r›   z'_multi_tensor_adamw.<locals>.<listcomp>²  r¦   r)   c                 óP   — g | ]#\  }}|d k                          |j        ¦  «        ‘Œ$S r¨   r©   rª   s      r(   r›   z'_multi_tensor_adamw.<locals>.<listcomp>¾  r­   r)   c                 ó6   — g | ]}|                      ¦   «         ‘ŒS r•   r¯   r°   s     r(   r›   z'_multi_tensor_adamw.<locals>.<listcomp>¿  r±   r)   )rH   r	   rB   Ú_foreach_negÚtupleÚ_foreach_add_Ú_foreach_mul_Ú_foreach_addcmul_Ú_foreach_sub_Ú_foreach_neg_Ú_foreach_divÚ_foreach_reciprocal_Ú_foreach_sqrtrh   Ú_foreach_div_Ú_foreach_mulÚ_foreach_addÚzipÚ_foreach_addcdiv_)r   rO   rP   rQ   rR   rS   r   r>   r?   r   r   r   r   r   r   r@   rŠ   rŒ   r�   rŽ   r�   r‹   Úeps_over_step_sizer‘   ÚmasksÚ
mask_scales          ```                r(   rn   rn   L  sù  øøø€ õ& ˆ6�{„{�aÒÐØˆàð 7Ý! &¨+Ñ6Ô6Ð6àð 1ÝÔ"¥5¨¡<¤<Ñ0Ô0ˆàPÐPÈ%ÐPÑPÔP€EØVÐVÈXÐVÑVÔV€HØ\Ð\ÐP[Ð\Ñ\Ô\€KØRÐRÈ6ÐRÑRÔR€Fõ 
Ô˜ QÑ'Ô'Ð'ð �^ˆrˆr¨¨q©°6Ñ)9€HÝ	Ô˜  X°Ñ%<Ñ <Ñ=Ô=Ð=õ 
Ô˜ %Ñ(Ô(Ð(Ý	Ô˜ %¨q°5©yÐ9Ñ9Ô9Ð9å	Ô˜ UÑ+Ô+Ð+Ý	Ô˜K¨°°q¸5±yÑAÔAÐAàñ M9àKÐKÐKÐK¸{ÐKÑKÔKÐØKÐKÐKÐK¸{ÐKÑKÔKÐåÔÐ,¨aÑ0Ô0Ð0ÝÔÐ,¨aÑ0Ô0Ð0ÝÔÐ,Ñ-Ô-Ð-ÝÔÐ,Ñ-Ô-Ð-õ Ô&Ð'7¸Ñ<Ô<ˆ	ÝÔ" 9Ñ-Ô-Ð-ÝÔ˜IÑ&Ô&Ð&å %Ô 3Ð4DÑ EÔ EÐàð 	:àlÐlÐ\kÐlÑlÔlˆOÝÔ# O°[ÑAÔAÐAÝÔ,¨_Ñ=Ô=ˆJˆJåÔ,¨[Ñ9Ô9ˆJåÔØÝÔÐ4°iÑ@Ô@ñ	
ô 	
ð 	
õ #Ô/°	¸3Ñ?Ô?ÐÝÔ"Ð#5Ñ6Ô6Ð6ÝÔ" :Ð/AÑBÔBˆàð 	;åÔ& x°Ñ7Ô7ˆEØGÐGµS¸ÀÑ5FÔ5FÐGÑGÔGˆEØ2Ð2¨EÐ2Ñ2Ô2ˆJÝÔ# J°Ñ5Ô5Ð5åÔ  zÑ2Ô2Ð2ÝÔ)¨(°EÑ:Ô:ˆHåÔ ¨°%Ñ8Ô8Ð8Ð8Ð8àRÐRÐRÐRÀkÐRÑRÔRÐØRÐRÐRÐRÀkÐRÑRÔRÐà?Ð?Ð?Ð?Ð.>Ð?Ñ?Ô?ˆ	à FÐ FÐ5EÐ FÑ FÔ FÐàð 	5àlÐlÐ\kÐlÑlÔlˆOÝÔ# O°[ÑAÔAÐAÝÔ'¨Ñ8Ô8ˆEˆEåÔ'¨Ñ4Ô4ˆEåÔ˜EÐ#8Ñ9Ô9Ð9ÝÔ˜E 3Ñ'Ô'Ð'àð 	;åÔ& x°Ñ7Ô7ˆEØGÐGµS¸ÀÑ5FÔ5FÐGÑGÔGˆEØ2Ð2¨EÐ2Ñ2Ô2ˆJÝÔ# J°Ñ5Ô5Ð5åÔ  zÑ2Ô2Ð2ÝÔ)¨(°EÑ:Ô:ˆHõ
 	Ô˜E 9Ñ-Ô-Ð-ÝÔ ¨°%Ñ8Ô8Ð8Ð8Ð8r)   )NF)rX   Útypingr   r   r   rB   r   Útorch.optim.optimizerr   Ú_helpersr	   r
   r   Ú_typesr   r   rZ   rY   rK   ro   rn   r•   r)   r(   ú<module>rÔ      s~  ððð ð )Ð (Ð (Ð (Ð (Ð (Ð (Ð (Ð (Ð (à €€€Ø Ð Ð Ð Ð Ð Ø +Ð +Ð +Ð +Ð +Ð +à IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ Ð Ð Ð Ð Ð ð]ð ]ð ]ð ]ð ]�)ñ ]ô ]ð ]ðN #'Ø ð=ð =Ø�V”ð=à�FŒ|ð=ð �v”,ð=ð ˜&”\ð	=ð
 ˜fœð=ð ˜&”\ð=ð ˜$”ð=ð ð=ð ð=ð ð=ð ð=ð ð=ð ð=ð ð=ð  ð!=ð" ð#=ð$ ˜”ð%=ð& 
ð'=ð =ð =ð =ð@U=Ø�V”ðU=à�FŒ|ðU=ð �v”,ðU=ð ˜&”\ð	U=ð
 ˜fœðU=ð ˜&”\ðU=ð ðU=ð ðU=ð ðU=ð ðU=ð ðU=ð ðU=ð ðU=ð ðU=ð  ð!U=ð" ˜”ð#U=ð U=ð U=ð U=ðp}9Ø�V”ð}9à�FŒ|ð}9ð �v”,ð}9ð ˜&”\ð	}9ð
 ˜fœð}9ð ˜&”\ð}9ð ð}9ð ð}9ð ð}9ð ð}9ð ð}9ð ð}9ð ð}9ð ð}9ð  ð!}9ð" ˜”ð#}9ð }9ð }9ð }9ð }9ð }9r)   