§
    ·¯¸j<  ã                   óB   — d Z ddlZ G d„ dej        j        ¦  «        ZdS )z› AdaHessian Optimizer

Lifted from https://github.com/davda54/ada-hessian/blob/master/ada_hessian.py
Originally licensed MIT, Copyright 2020, David Samuel
é    Nc                   ó¸   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 dˆ fd	„	Zed
„ ¦   «         Zd„ Zd„ Z e	j
        ¦   «         d„ ¦   «         Z e	j
        ¦   «         dd„¦   «         Zˆ xZS )Ú
Adahessianaî  
    Implements the AdaHessian algorithm from "ADAHESSIAN: An Adaptive Second OrderOptimizer for Machine Learning"

    Arguments:
        params (iterable): iterable of parameters to optimize or dicts defining parameter groups
        lr (float, optional): learning rate (default: 0.1)
        betas ((float, float), optional): coefficients used for computing running averages of gradient and the
            squared hessian trace (default: (0.9, 0.999))
        eps (float, optional): term added to the denominator to improve numerical stability (default: 1e-8)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0.0)
        hessian_power (float, optional): exponent of the hessian trace (default: 1.0)
        update_each (int, optional): compute the hessian trace approximation only after *this* number of steps
            (to save time) (default: 1)
        n_samples (int, optional): how many times to sample `z` for the approximation of the hessian trace (default: 1)
    çš™™™™™¹?©gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?ç:Œ0âŽyE>ç        ç      ð?é   Fc
                 óÌ  •— d|k    st          d|› �¦  «        ‚d|k    st          d|› �¦  «        ‚d|d         cxk    rdk     sn t          d|d         › �¦  «        ‚d|d         cxk    rdk     sn t          d|d         › �¦  «        ‚d|cxk    rdk    sn t          d	|› �¦  «        ‚|| _        || _        |	| _        d
| _        t          j        ¦   «                              | j        ¦  «        | _        t          |||||¬¦  «        }
t          t          | ¦  «                             ||
¦  «         |                      ¦   «         D ]}d|_        d| j        |         d<   Œd S )Nr   zInvalid learning rate: zInvalid epsilon value: r   r	   z#Invalid beta parameter at index 0: r
   z#Invalid beta parameter at index 1: zInvalid Hessian power value: iÿÿÿ)ÚlrÚbetasÚepsÚweight_decayÚhessian_powerúhessian step)Ú
ValueErrorÚ	n_samplesÚupdate_eachÚavg_conv_kernelÚseedÚtorchÚ	GeneratorÚmanual_seedÚ	generatorÚdictÚsuperr   Ú__init__Ú
get_paramsÚhessÚstate)ÚselfÚparamsr   r   r   r   r   r   r   r   ÚdefaultsÚpÚ	__class__s               €úe/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/optim/adahessian.pyr   zAdahessian.__init__   s¸  ø€ ð �bŠyˆyÝÐ;°rÐ;Ð;Ñ<Ô<Ð<Ø�cŠzˆzÝÐ<°sÐ<Ð<Ñ=Ô=Ð=Ø�e˜A”hÐ$Ð$Ò$Ð$ Ò$Ð$Ð$Ð$ÝÐMÀ5ÈÄ8ÐMÐMÑNÔNÐNØ�e˜A”hÐ$Ð$Ò$Ð$ Ò$Ð$Ð$Ð$ÝÐMÀ5ÈÄ8ÐMÐMÑNÔNÐNØ�mÐ*Ð*Ò*Ð* sÒ*Ð*Ð*Ð*ÝÐL¸]ÐLÐLÑMÔMÐMà"ˆŒØ&ˆÔØ.ˆÔð ˆŒ	ÝœÑ*Ô*×6Ò6°t´yÑAÔAˆŒåØØØØ%Ø'ð
ñ 
ô 
ˆõ 	�j˜$ÑÔ×(Ò(¨°Ñ:Ô:Ð:à—’Ñ"Ô"ð 	.ð 	.ˆAØˆAŒFØ,-ˆDŒJ�qŒM˜.Ñ)Ð)ð	.ð 	.ó    c                 ó   — dS )NT© ©r!   s    r&   Úis_second_orderzAdahessian.is_second_orderF   s   € àˆtr'   c                 ó$   — d„ | j         D ¦   «         S )zH
        Gets all parameters in all param_groups with gradients
        c              3   ó>   K  — | ]}|d          D ]}|j         ¯	|V — ŒŒdS )r"   N)Úrequires_grad)Ú.0Úgroupr$   s      r&   ú	<genexpr>z(Adahessian.get_params.<locals>.<genexpr>O   s:   è è € Ð]Ð]�e¸%À¼/Ð]Ð]°QÈQÌ_Ð]�Ð]Ð]Ð]Ð]Ð]Ð]Ð]r'   )Úparam_groupsr*   s    r&   r   zAdahessian.get_paramsJ   s   € ð
 ^Ð]˜tÔ0Ð]Ñ]Ô]Ð]r'   c                 óØ   — |                       ¦   «         D ]T}t          |j        t          ¦  «        s8| j        |         d         | j        z  dk    r|j                             ¦   «          ŒUdS )z;
        Zeros out the accumulated hessian traces.
        r   r   N)r   Ú
isinstancer   Úfloatr    r   Úzero_)r!   r$   s     r&   Úzero_hessianzAdahessian.zero_hessianQ   sj   € ð
 —’Ñ"Ô"ð 	ð 	ˆAÝ˜aœf¥eÑ,Ô,ð °´¸A´¸~Ô1NÐQUÔQaÑ1aÐefÒ1fÐ1fØ”—’‘”�øð	ð 	r'   c           
      óö  ‡ — g }t          d„ ‰                      ¦   «         ¦  «        D ]Q}‰ j        |         d         ‰ j        z  dk    r|                     |¦  «         ‰ j        |         dxx         dz  cc<   ŒRt          |¦  «        dk    rdS ‰ j        j        |d         j        k    r<t          j	        |d         j        ¦  «         
                    ‰ j        ¦  «        ‰ _        d„ |D ¦   «         }t          ‰ j        ¦  «        D ]s}ˆ fd„|D ¦   «         }t          j                             |||d|‰ j        dz
  k     ¬	¦  «        }t!          |||¦  «        D ]!\  }}}|xj        ||z  ‰ j        z  z  c_        Œ"ŒtdS )
z}
        Computes the Hutchinson approximation of the hessian trace and accumulates it for each trainable parameter.
        c                 ó   — | j         d uS ©N©Úgrad)r$   s    r&   ú<lambda>z(Adahessian.set_hessian.<locals>.<lambda>a   s   €  !¤&°Ð"4€ r'   r   r   r
   Nc                 ó   — g | ]	}|j         ‘Œ
S r)   r;   )r/   r$   s     r&   ú
<listcomp>z*Adahessian.set_hessian.<locals>.<listcomp>l   s   € Ð(Ð(Ð(˜A�”Ð(Ð(Ð(r'   c           	      ó†   •— g | ]=}t          j        d d|                     ¦   «         ‰j        |j        ¬¦  «        dz  dz
  ‘Œ>S )r   é   )r   Údeviceg       @r	   )r   ÚrandintÚsizer   rB   )r/   r$   r!   s     €r&   r?   z*Adahessian.set_hessian.<locals>.<listcomp>p   sI   ø€ ÐwÐwÐwÐkl•%”-  1 a§f¢f¡h¤h¸$¼.ÐQRÔQYÐZÑZÔZÐ]`Ñ`ÐcfÑfÐwÐwÐwr'   T)Úgrad_outputsÚonly_inputsÚretain_graph)Úfilterr   r    r   ÚappendÚlenr   rB   r   r   r   r   Úranger   Úautogradr<   Úzipr   )	r!   r"   r$   ÚgradsÚiÚzsÚh_zsÚh_zÚzs	   `        r&   Úset_hessianzAdahessian.set_hessianZ   s¦  ø€ ð ˆÝÐ4Ð4°d·o²oÑ6GÔ6GÑHÔHð 	/ð 	/ˆAØŒz˜!Œ}˜^Ô,¨tÔ/?Ñ?À1ÒDÐDØ—’˜aÑ Ô Ð ØŒJ�qŒM˜.Ð)Ð)Ô)¨QÑ.Ð)Ð)Ñ)Ð)åˆv‰;Œ;˜!ÒÐØˆFàŒ>Ô  F¨1¤IÔ$4Ò4Ð4Ý"œ_¨V°A¬YÔ-=Ñ>Ô>×JÒJÈ4Ì9ÑUÔUˆDŒNà(Ð( Ð(Ñ(Ô(ˆå�t”~Ñ&Ô&ð 	3ð 	3ˆAàwÐwÐwÐwÐpvÐwÑwÔwˆBÝ”>×&Ò&Ø�v¨B¸DÈqÐSWÔSaÐdeÑSeÒOeð 'ñ gô gˆDå   r¨6Ñ2Ô2ð 3ð 3‘	��Q˜Ø�”˜# ™' D¤NÑ2Ñ2�”�ð3ð	3ð 	3r'   Nc                 óÔ  — d}|�
 |¦   «         }|                       ¦   «          |                      ¦   «          | j        D �](}|d         D �]}|j        �|j        €Œ| j        rw|                     ¦   «         dk    r_t          j        |j        ¦  «         	                    ddgd¬¦  «         
                    |j        ¦  «                             ¦   «         |_        |                     d|d	         |d
         z  z
  ¦  «         | j        |         }t          |¦  «        dk    r3d|d<   t          j        |¦  «        |d<   t          j        |¦  «        |d<   |d         |d         }}|d         \  }}	|dxx         dz  cc<   |                     |¦  «                             |j        d|z
  ¬¦  «         |                     |	¦  «                             |j        |j        d|	z
  ¬¦  «         d||d         z  z
  }
d|	|d         z  z
  }|d         }||z                       |dz  ¦  «                             |d         ¦  «        }|d	         |
z  }|                     ||| ¬¦  «         �Œ�Œ*|S )z¿
        Performs a single optimization step.
        Arguments:
            closure (callable, optional) -- a closure that reevaluates the model and returns the loss (default: None)
        Nr"   é   rA   é   T)ÚdimÚkeepdimr
   r   r   r   ÚstepÚexp_avgÚexp_hessian_diag_sqr   )Úalpha)Úvaluer   r   )r7   rT   r2   r<   r   r   rX   r   ÚabsÚmeanÚ	expand_asÚcloneÚmul_r    rJ   Ú
zeros_likeÚadd_Úaddcmul_Úpow_Úaddcdiv_)r!   ÚclosureÚlossr0   r$   r    r[   r\   Úbeta1Úbeta2Úbias_correction1Úbias_correction2ÚkÚdenomÚ	step_sizes                  r&   rZ   zAdahessian.stepv   s…  € ð ˆØÐØ�7‘9”9ˆDà×ÒÑÔÐØ×ÒÑÔÐàÔ&ð %	=ñ %	=ˆEØ˜8”_ð $=ñ $=�Ø”6�> Q¤V ^ØàÔ'ð h¨A¯EªE©G¬G°qªL¨LÝ"œY q¤vÑ.Ô.×3Ò3¸¸A¸ÈÐ3ÑMÔM×WÒWÐXYÔX^Ñ_Ô_×eÒeÑgÔg�A”Fð —’�q˜5 œ;¨¨~Ô)>Ñ>Ñ>Ñ?Ô?Ð?àœ
 1œ�õ �u‘:”: ’?�?Ø$%�E˜&‘Må',Ô'7¸Ñ':Ô':�E˜)Ñ$å38Ô3CÀAÑ3FÔ3F�EÐ/Ñ0à/4°YÔ/?ÀÐG\ÔA]Ð,�Ø$ Wœ~‘��uØ�f��” Ñ"��‘ð —’˜UÑ#Ô#×(Ò(¨¬°q¸5±yÐ(ÑAÔAÐAØ#×(Ò(¨Ñ/Ô/×8Ò8¸¼ÀÄÈqÐSXÉyÐ8ÑYÔYÐYà#$ u°°f´Ñ'=Ñ#=Ð Ø#$ u°°f´Ñ'=Ñ#=Ð à˜/Ô*�Ø,Ð/?Ñ?×EÒEÀaÈ!ÁeÑLÔL×QÒQÐRWÐX]ÔR^Ñ_Ô_�ð " $œKÐ*:Ñ:�	Ø—
’
˜7 E°)°�
Ñ<Ô<Ð<Ñ<ñI$=ðL ˆr'   )r   r   r   r   r	   r
   r
   Fr:   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Úpropertyr+   r   r7   r   Úno_gradrT   rZ   Ú__classcell__)r%   s   @r&   r   r   	   sé   ø€ € € € € ðð ð& ØØØØØØØ!ð*.ð *.ð *.ð *.ð *.ð *.ðX ðð ñ „Xðð^ð ^ð ^ðð ð ð €U„]�_„_ð3ð 3ñ „_ð3ð6 €U„]�_„_ð5ð 5ð 5ñ „_ð5ð 5ð 5ð 5ð 5r'   r   )ru   r   ÚoptimÚ	Optimizerr   r)   r'   r&   ú<module>r{      s]   ððð ð
 €€€ðcð cð cð cð c�”Ô&ñ cô cð cð cð cr'   