§
    ·¯¸j  ã            $       óð   — d Z ddlZddlmZmZ ddlZddlmZ ddlm	Z	 dej
        dej
        d	ej
        d
ej
        dededededej
        dedededededededeeef         def$d„Z G d„ de¦  «        ZdS )aÉ   PyTorch MARS Optimizer

Code simplified from https://github.com/AGI-Arena/MARS

Paper: MARS: Unleashing the Power of Variance Reduction for Training Large Models - https://arxiv.org/abs/2411.10438

@article{yuan2024mars,
  title={MARS: Unleashing the Power of Variance Reduction for Training Large Models},
  author={Yuan, Huizhuo and Liu, Yifeng and Wu, Shuang and Zhou, Xun and Gu, Quanquan},
  journal={arXiv preprint arXiv:2411.10438},
  year={2024}
}
é    N)ÚOptionalÚTuple)Ú	Optimizeré   )ÚParamsTÚpÚgradÚexp_avgÚ
exp_avg_sqÚlrÚweight_decayÚbeta1Úbeta2Ú	last_gradÚepsÚstepÚgammaÚ	mars_typeÚ
is_grad_2dÚoptimize_1dÚlr_1d_factorÚbetas_1dÚcautionc                 ó2  — |s|�rÇd|z
  }|
dk    r|}nP||z
                        |||z  z  ¦  «                             |¦  «        }t          j        |¦  «        }|dk    r||z  }|                      |¦  «                             ||¬¦  «         |ra||z  dk                         |j        ¦  «        }|                     |                     ¦   «                              d¬¦  «        ¦  «         ||z  }|dk    r™|                      |¦  «         	                    ||d|z
  ¬¦  «         d||
z  z
  }d||
z  z
  }| 
                    ¦   «         t          j
        |¦  «        z                       |	¦  «        }| |z  ||z                       |¦  «        z   }n#|d	k    r| |z  |                     ¦   «         z   }nJ ‚|                      || ¬¦  «         �nH|\  }}|                      |¦  «                             |d|z
  ¬¦  «         |                      |¦  «         	                    ||d|z
  ¬¦  «         d||
z  z
  }d||
z  z
  }| 
                    ¦   «         t          j
        |¦  «        z                       |	¦  «        }|ra||z  dk                         |j        ¦  «        }|                     |                     ¦   «                              d¬¦  «        ¦  «         ||z  }| |z  ||z                       |¦  «        z   }|                      |||z   ¬¦  «         ||fS )
Nç      ð?r   )Úalphar   gü©ñÒMbP?)ÚminÚadamw)ÚvalueÚlion)Úmul_Úadd_ÚtorchÚnormÚtoÚdtypeÚdiv_ÚmeanÚclamp_Úaddcmul_ÚsqrtÚmathÚsign)r   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   Úone_minus_beta1Úc_tÚc_t_normÚmaskÚbias_correction1Úbias_correction2ÚdenomÚupdateÚbeta1_1dÚbeta2_1ds                               ú_/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/optim/mars.pyÚ_mars_single_tensor_stepr9      s6  € ð* ð )3�jñ )3Ø˜u™*ˆØ�1Š9ˆ9àˆCˆCà˜)Ñ#×)Ò)¨%°5¸?Ñ3JÑ*KÑLÔL×QÒQÐRVÑWÔWˆCÝ”z #‘”ˆHØ˜"Š}ˆ}Ø˜H‘n�Ø�Š�UÑÔ× Ò  ¨OÐ Ñ<Ô<Ð<àð 	%à˜d‘N QÒ&×*Ò*¨4¬:Ñ6Ô6ˆDØ�IŠI�d—i’i‘k”k×(Ò(¨TÐ(Ñ2Ô2Ñ3Ô3Ð3Ø ‘nˆGà˜ÒÐØ�OŠO˜EÑ"Ô"×+Ò+¨C°¸BÀ¹JÐ+ÑGÔGÐGØ" U¨d¡]Ñ2ÐØ" U¨d¡]Ñ2ÐØ—_’_Ñ&Ô&­¬Ð3CÑ)DÔ)DÑD×JÒJÈ3ÑOÔOˆEØ˜Ñ%¨Ð3CÑ)C×(IÒ(IÈ%Ñ(PÔ(PÑPˆFˆFØ˜&Ò Ð Ø˜Ñ%¨¯ª©¬Ñ6ˆFˆFà�5Ø	�Šˆv˜b˜SˆÑ!Ô!Ð!Ñ!à%Ñˆ�(Ø�Š�XÑÔ×#Ò# D°°X±Ð#Ñ>Ô>Ð>Ø�Š˜Ñ!Ô!×*Ò*¨4°¸RÀ(¹]Ð*ÑKÔKÐKØ ¨TÑ!1Ñ1ÐØ ¨TÑ!1Ñ1ÐØ—’Ñ"Ô"¥T¤YÐ/?Ñ%@Ô%@Ñ@×FÒFÀsÑKÔKˆØð 	%Ø˜d‘N QÒ&×*Ò*¨4¬:Ñ6Ô6ˆDØ�IŠI�d—i’i‘k”k×(Ò(¨TÐ(Ñ2Ô2Ñ3Ô3Ð3Ø ‘nˆGØ�\Ñ! WÐ/?Ñ%?×$EÒ$EÀeÑ$LÔ$LÑLˆØ	�Šˆv˜r LÑ0Ð1ˆÑ2Ô2Ð2Ø�JÐÐó    c                   óÎ   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 ddededeeef         dedededededede	eeef                  defˆ fd„Z
ˆ fd„Z ej        ¦   «         dd„¦   «         Zˆ xZS )ÚMarsz• MARS Optimizer

    Paper: MARS: Unleashing the Power of Variance Reduction for Training Large Models
        https://arxiv.org/abs/2411.10438

    çú~j¼t“h?©gÍÌÌÌÌÌì?g®Gáz®ï?ç:Œ0âŽyE>ç        çš™™™™™™?r   Fr   NÚparamsr   Úbetasr   r   r   r   r   r   r   r   c                 ó@  •— d|k    s"t          d                     |¦  «        ¦  «        ‚d|k    s"t          d                     |¦  «        ¦  «        ‚d|d         cxk    rdk     s*n t          d                     |d         ¦  «        ¦  «        ‚d|d         cxk    rdk     s*n t          d                     |d         ¦  «        ¦  «        ‚|d	v s
J d
¦   «         ‚t          ||||||||	|
p||¬¦
  «
        }t          t          | ¦  «                             ||¦  «         d S )Nr@   zInvalid learning rate: {}zInvalid epsilon value: {}r   r   z%Invalid beta parameter at index 0: {}r   z%Invalid beta parameter at index 1: {})r   r    zMARS type not supported)
r   rC   r   r   r   r   r   r   r   r   )Ú
ValueErrorÚformatÚdictÚsuperr<   Ú__init__)ÚselfrB   r   rC   r   r   r   r   r   r   r   r   ÚdefaultsÚ	__class__s                €r8   rI   zMars.__init__b   sU  ø€ ð �bŠyˆyÝÐ8×?Ò?ÀÑCÔCÑDÔDÐDØ�cŠzˆzÝÐ8×?Ò?ÀÑDÔDÑEÔEÐEØ�e˜A”hÐ$Ð$Ò$Ð$ Ò$Ð$Ð$Ð$ÝÐD×KÒKÈEÐRSÌHÑUÔUÑVÔVÐVØ�e˜A”hÐ$Ð$Ò$Ð$ Ò$Ð$Ð$Ð$ÝÐD×KÒKÈEÐRSÌHÑUÔUÑVÔVÐVØÐ-Ð-Ð-Ð-Ð/HÑ-Ô-Ð-åØØØØ%ØØØ#Ø%ØÐ& Øð
ñ 
ô 
ˆõ 	�d�DÑÔ×"Ò" 6¨8Ñ4Ô4Ð4Ð4Ð4r:   c                 ó˜   •— t          t          | ¦  «                             |¦  «         | j        D ]}|                     dd¦  «         Œd S )Nr   F)rH   r<   Ú__setstate__Úparam_groupsÚ
setdefault)rJ   ÚstateÚgrouprL   s      €r8   rN   zMars.__setstate__ˆ   sV   ø€ Ý�d�DÑÔ×&Ò& uÑ-Ô-Ð-ØÔ&ð 	/ð 	/ˆEØ×Ò˜Y¨Ñ.Ô.Ð.Ð.ð	/ð 	/r:   c                 ó@  — d}|�5t          j        ¦   «         5   |¦   «         }ddd¦  «         n# 1 swxY w Y   | j        D �][}|d         D �]N}|j        €Œ|j        }|j        rt          d¦  «        ‚| j        |         }t          |¦  «        dk    rJd|d<   t          j        |¦  «        |d<   t          j        |¦  «        |d<   t          j        |¦  «        |d	<   |dxx         dz  cc<   |d         }|d         }|d	         }	|d         }
|d
         }|d         }|d         \  }}|j	        dk    }t          ||||	|||||
|d         ||d         |d         ||d         |d         |d         |d         ¬¦  «         |d                              |¦  «         �ŒP�Œ]|S )z¶Performs a single optimization step.

        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        NrB   zJAdam does not support sparse gradients, please consider SparseAdam insteadr   r   r   r
   r   r   r   r   rC   é   r   r   r   r   r   r   r   )r   r   r   r   r   r   )r#   Úenable_gradrO   r	   Ú	is_sparseÚRuntimeErrorrQ   ÚlenÚ
zeros_likeÚndimr9   Úcopy_)rJ   ÚclosureÚlossrR   r   r	   rQ   r   r
   r   r   r   Úwdr   r   r   s                   r8   r   z	Mars.step�   s6  € ð ˆØÐÝÔ"Ñ$Ô$ð !ð !Ø�w‘y”y�ð!ð !ð !ñ !ô !ð !ð !ð !ð !ð !ð !øøøð !ð !ð !ð !ð Ô&ð 6	/ñ 6	/ˆEØ˜8”_ð 5/ñ 5/�Ø”6�>ØØ”v�Ø”>ð uÝ&Ð'sÑtÔtÐtàœ
 1œ�å�u‘:”: ’?�?Ø$%�E˜&‘Må',Ô'7¸Ñ':Ô':�E˜)Ñ$å).Ô)9¸!Ñ)<Ô)<�E˜+Ñ&å*/Ô*:¸1Ñ*=Ô*=�E˜,Ñ'à�f��” Ñ"��‘Ø˜V”}�Ø 	Ô*�Ø" <Ô0�
Ø! +Ô.�	Ø˜4”[�Ø˜>Ô*�Ø$ Wœ~‘��uØ!œY¨!š^�
õ )ØØØØØØØØØØ˜%”LØØ˜'”NØ# KÔ0Ø)Ø % mÔ 4Ø!& ~Ô!6Ø" :Ô.Ø! )Ô,ð%ñ ô ð ð0 �kÔ"×(Ò(¨Ñ.Ô.Ð.Ñ.ñk5/ðn ˆs   ˜/¯3¶3)
r=   r>   r?   r@   rA   r   Fr   NF)N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Úfloatr   ÚstrÚboolr   rI   rN   r#   Úno_gradr   Ú__classcell__)rL   s   @r8   r<   r<   [   s9  ø€ € € € € ðð ð Ø)4ØØ"$Ø Ø$Ø %Ø"%Ø6:Ø!ð$5ð $5àð$5ð ð$5ð ˜ ˜Ô&ð	$5ð
 ð$5ð  ð$5ð ð$5ð ð$5ð ð$5ð  ð$5ð ˜u U¨E \Ô2Ô3ð$5ð ð$5ð $5ð $5ð $5ð $5ð $5ðL/ð /ð /ð /ð /ð
 €U„]�_„_ðDð Dð Dñ „_ðDð Dð Dð Dð Dr:   r<   )rb   r,   Útypingr   r   r#   Útorch.optim.optimizerr   Ú_typesr   ÚTensorrc   Úintrd   re   r9   r<   © r:   r8   ú<module>rn      su  ððð ð €€€Ø "Ð "Ð "Ð "Ð "Ð "Ð "Ð "à €€€Ø +Ð +Ð +Ð +Ð +Ð +à Ð Ð Ð Ð Ð ð?ØŒ<ð?àŒlð?ð ”ð?ð ”Lð	?ð
 ð?ð ð?ð ð?ð ð?ð ”<ð?ð ð?ð ð?ð ð?ð ð?ð ð?ð ð?ð  ð!?ð" ˜˜u˜Ô%ð#?ð$ ð%?ð ?ð ?ð ?ðDwð wð wð wð wˆ9ñ wô wð wð wð wr:   