§
    ·¯¸jñ&  ã                   óf   — d Z ddlZddlmZmZ ddlZddlmZ  G d„ dej        j	        ¦  «        Z
dS )zí Adafactor Optimizer

Lifted from https://github.com/pytorch/fairseq/blob/master/fairseq/optim/adafactor.py

Modified by Ross Wightman to fix some issues with factorization dims for non nn.Linear layers

Original header/copyright below.
é    N)ÚOptionalÚTupleé   )ÚParamsTc                   ó  ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 ddedee         dededededeeeef                  dededede	defˆ fd„Z
ˆ fd„Zed„ ¦   «         Zedd„¦   «         Zed„ ¦   «         Zd„ Z ej        ¦   «         d d„¦   «         Zˆ xZS )!Ú	AdafactoraÕ  Implements Adafactor algorithm.

    This implementation is based on: `Adafactor: Adaptive Learning Rates with Sublinear Memory Cost`
    (see https://arxiv.org/abs/1804.04235)

    Note that this optimizer internally adjusts the learning rate depending on the
    *scale_parameter*, *relative_step* and *warmup_init* options.

    To use a manual (external) learning rate schedule you should set `scale_parameter=False` and
    `relative_step=False`.

    Ags:
        params: iterable of parameters to optimize or dicts defining parameter groups
        lr: external learning rate
        eps: regularization constants for square gradient and parameter scale respectively
        eps_scale: regularization constants for parameter scale respectively
        clip_threshold: threshold of root-mean-square of final gradient update
        decay_rate: coefficient used to compute running averages of square gradient
        beta1: coefficient used for computing running averages of gradient
        weight_decay: weight decay
        scale_parameter: if True, learning rate is scaled by root-mean-square of parameter
        warmup_init: time-dependent learning rate computation depends on whether warm-up initialization is being used
    Nç ÂëþKH´9çü©ñÒMbP?ç      ð?çš™™™™™é¿ç        TFé   ÚparamsÚlrÚepsÚ	eps_scaleÚclip_thresholdÚ
decay_rateÚbetasÚweight_decayÚscale_parameterÚwarmup_initÚmin_dim_size_to_factorÚcautionc                 óÔ   •— | }|
r|st          d¦  «        ‚|€d n|d         }t          ||||||||	||
||¬¦  «        }t          t          | ¦  «                             ||¦  «         d S )Nz'warmup_init requires relative_step=Truer   )r   r   r   r   r   Úbeta1r   r   Úrelative_stepr   r   r   )Ú
ValueErrorÚdictÚsuperr   Ú__init__)Úselfr   r   r   r   r   r   r   r   r   r   r   r   r   r   ÚdefaultsÚ	__class__s                   €úd/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/optim/adafactor.pyr!   zAdafactor.__init__.   s�   ø€ ð ˜ˆØð 	H˜}ð 	HÝÐFÑGÔGÐGà˜��¨5°¬8ˆÝØØØØ)Ø!ØØ%Ø+Ø'Ø#Ø#9Øð
ñ 
ô 
ˆõ 	�i˜ÑÔ×'Ò'¨°Ñ9Ô9Ð9Ð9Ð9ó    c                 ó¶   •— t          ¦   «                              |¦  «         | j        D ].}|                     dd¦  «         |                     dd¦  «         Œ/d S )Nr   Fr   r   )r    Ú__setstate__Úparam_groupsÚ
setdefault)r"   ÚstateÚgroupr$   s      €r%   r(   zAdafactor.__setstate__R   sg   ø€ Ý‰Œ×Ò˜UÑ#Ô#Ð#ØÔ&ð 	;ð 	;ˆEØ×Ò˜Y¨Ñ.Ô.Ð.Ø×ÒÐ5°rÑ:Ô:Ð:Ð:ð	;ð 	;r&   c                 óþ   — | d         rn| d         rd|d         z  nd}t          |dt          j        |d         ¦  «        z  ¦  «        }d}| d         rt          | d         |d	         ¦  «        }||z  | d
<   | d
         S )Nr   r   g�íµ ÷Æ°>Ústepg{®Gáz„?r   r   r   ÚRMSr   )ÚminÚmathÚsqrtÚmax)Úparam_groupÚparam_stateÚmin_stepÚlr_tÚparam_scales        r%   Ú_get_lrzAdafactor._get_lrX   s˜   € à�Ô'ð 	3Ø5@ÀÔ5OÐY�t˜k¨&Ô1Ñ1Ð1ÐUYˆHÝ�x ¥t¤y°¸VÔ1DÑ'EÔ'EÑ!EÑFÔFˆDØˆKØÐ,Ô-ð PÝ! +¨kÔ":¸KÈÔ<NÑOÔO�Ø $ {Ñ 2ˆK˜ÑØ˜4Ô Ð r&   c                 óÒ   — | d         d u}d }t          |¦  «        }|dk    r|d         |k    r|d         |k    rd}n(|dk    r"|d         |k    r|d         |k    r
|dz
  |dz
  f}||fS )Nr   é   r   r   )r   r   éþÿÿÿéÿÿÿÿ)Úlen)r4   Úparam_shapeÚmin_size_to_factorÚuse_first_momentÚfactoredÚndims         r%   Ú_get_optionszAdafactor._get_optionsc   sš   € à& wÔ/°tÐ;ÐØˆÝ�;ÑÔˆð
 �!Š8ˆ8˜ AœÐ);Ò;Ð;ÀÈAÄÐQcÒ@cÐ@càˆHˆHØ�QŠYˆY˜; rœ?Ð-?Ò?Ð?ÀKÐPRÄOÐVhÒDhÐDhà˜a‘x ¨¡Ð)ˆHàÐ)Ð)Ð)r&   c                 ó\   — |                       d¦  «        |                      ¦   «         dz  z  S )Nr;   g      à?)ÚnormÚnumel)Útensors    r%   Ú_rmszAdafactor._rmsu   s$   € à�{Š{˜1‰~Œ~ §¢¡¤°3Ñ!6Ñ7Ð7r&   c                 óø   — ||                      |d¬¦  «        z                       ¦   «                              |¦  «        }|                     |¦  «                             ¦   «         }t	          j        ||¦  «        S )NT)ÚdimÚkeepdim)ÚmeanÚrsqrt_Ú	unsqueezeÚrsqrtÚtorchÚmul)r"   Úexp_avg_sq_rowÚexp_avg_sq_colÚdim_colÚdim_rowÚr_factorÚc_factors          r%   Ú_approx_sq_gradzAdafactor._approx_sq_grady   sk   € à" ^×%8Ò%8¸WÈdÐ%8Ñ%SÔ%SÑS×[Ò[Ñ]Ô]×gÒgÐhoÑpÔpˆØ!×+Ò+¨GÑ4Ô4×:Ò:Ñ<Ô<ˆÝŒy˜ 8Ñ,Ô,Ð,r&   c                 óà
  — d}|�5t          j        ¦   «         5   |¦   «         }ddd¦  «         n# 1 swxY w Y   | j        D �]+}|d         D �]}|j        €Œ|j        }|j        t           j        t           j        hv r|                     ¦   «         }|j        rt          d¦  «        ‚| j
        |         }|                      ||j        |d         ¬¦  «        \  }}t          |¦  «        dk    r¸d|d<   |rt          j        |¦  «        |d<   |�{|\  }	}
d	„ }t          j         ||j        |
¦  «        ¦  «                             |¦  «        |d
<   t          j         ||j        |	¦  «        ¦  «                             |¦  «        |d<   nt          j        |¦  «        |d<   d|d<   n}|r|d                              |¦  «        |d<   |�=|d
                              |¦  «        |d
<   |d                              |¦  «        |d<   n|d                              |¦  «        |d<   |}|j        t           j        t           j        hv r|                     ¦   «         }|dxx         dz  cc<   |                      |¦  «        |d<   |                      ||¦  «        }dt'          j        |d         |d         ¦  «        z
  }|dz  |d         z   }|�Å|\  }	}
|d
         }|d         }|                     |¦  «                             |                     |
¬¦  «        d|z
  ¬¦  «         |                     |¦  «                             |                     |	¬¦  «        d|z
  ¬¦  «         |                      |||	|
¦  «        }|                     |¦  «         n\|d         }|                     |¦  «                             |d|z
  ¬¦  «         |                     ¦   «                              |¦  «        }|                     |                      |¦  «        |d         z                       d¬¦  «        ¦  «         |                     |¦  «         |r­|d         }|                     |d         ¦  «                             |d|d         z
  ¬¦  «         |d         rb||z  dk                         |j        ¦  «        }|                     |                     ¦   «                              d¬¦  «        ¦  «         ||z  }n|}|d         dk    r!|                     ||d          |z  ¬¦  «         |                     | ¦  «         |j        t           j        t           j        hv r|                     |¦  «         �Œ �Œ-|S )z¥Performs a single optimization step.
        Arguments:
            closure (callable, optional): A closure that reevaluates the model and returns the loss.
        Nr   z,Adafactor does not support sparse gradients.r   )r@   r   r.   Úexp_avgc                 ó2   — | d |…         | |dz   d …         z   S )Nr   © )ÚshaperK   s     r%   Ú_remove_dimz#Adafactor.step.<locals>._remove_dim¤   s!   € Ø#(¨¨#¨¤;°°s¸Q±w°x°x´Ñ#@Ð@r&   rS   rT   Ú
exp_avg_sqr/   r   r   r   r;   r   )rK   )Úalphar   )r0   r   r   r
   r   )rQ   Úenable_gradr)   ÚgradÚdtypeÚfloat16Úbfloat16ÚfloatÚ	is_sparseÚRuntimeErrorr+   rD   r^   r>   Ú
zeros_likeÚzerosÚtorI   r9   r1   ÚpowÚmul_Úadd_rM   rY   rP   Údiv_Úclamp_Úcopy_)r"   ÚclosureÚlossr,   Úprc   r+   Úfactored_dimsrA   rU   rV   r_   Úp_fp32r7   Úbeta2tÚupdaterS   rT   r`   r[   Úmasks                        r%   r.   zAdafactor.step   sÍ  € ð ˆØÐÝÔ"Ñ$Ô$ð !ð !Ø�w‘y”y�ð!ð !ð !ñ !ô !ð !ð !ð !ð !ð !ð !øøøð !ð !ð !ð !ð Ô&ð Y	$ñ Y	$ˆEØ˜8”_ð X$ñ X$�Ø”6�>ØØ”v�Ø”:¥%¤-µ´Ð!@Ð@Ð@ØŸ:š:™<œ<�DØ”>ð WÝ&Ð'UÑVÔVÐVàœ
 1œ�à26×2CÒ2CØØ”JØ',Ð-EÔ'Fð 3Dñ 3ô 3Ñ/�Ð/õ �u‘:”: ’?�?Ø$%�E˜&‘Mà'ð Bå+0Ô+;¸DÑ+AÔ+A˜˜iÑ(Ø$Ð0Ø+8Ñ(˜ ðAð Að Aå27´+¸k¸kÈ$Ì*ÐV]Ñ>^Ô>^Ñ2_Ô2_×2bÒ2bÐcgÑ2hÔ2h˜Ð.Ñ/Ý27´+¸k¸kÈ$Ì*ÐV]Ñ>^Ô>^Ñ2_Ô2_×2bÒ2bÐcgÑ2hÔ2h˜Ð.Ñ/Ð/å.3Ô.>¸tÑ.DÔ.D˜˜lÑ+à#$�E˜%‘L�Là'ð EØ+0°Ô+;×+>Ò+>¸tÑ+DÔ+D˜˜iÑ(Ø$Ð0Ø27Ð8HÔ2I×2LÒ2LÈTÑ2RÔ2R˜Ð.Ñ/Ø27Ð8HÔ2I×2LÒ2LÈTÑ2RÔ2R˜Ð.Ñ/Ð/à.3°LÔ.A×.DÒ.DÀTÑ.JÔ.J˜˜lÑ+à�Ø”7�uœ}­e¬nÐ=Ð=Ð=Ø#Ÿ\š\™^œ^�Fà�f��” Ñ"��‘Ø#Ÿyšy¨Ñ0Ô0��e‘Ø—|’| E¨5Ñ1Ô1�à�tœx¨¨f¬°u¸\Ô7JÑKÔKÑK�Ø ™ U¨5¤\Ñ1�Ø Ð,Ø'4Ñ$�G˜WØ%*Ð+;Ô%<�NØ%*Ð+;Ô%<�Nà"×'Ò'¨Ñ/Ô/×4Ò4°V·[²[ÀW°[Ñ5MÔ5MÐUXÐ[aÑUaÐ4ÑbÔbÐbØ"×'Ò'¨Ñ/Ô/×4Ò4°V·[²[ÀW°[Ñ5MÔ5MÐUXÐ[aÑUaÐ4ÑbÔbÐbð "×1Ò1°.À.ÐRYÐ[bÑcÔc�FØ—K’K Ñ%Ô%Ð%Ð%à!& |Ô!4�Jà—O’O FÑ+Ô+×0Ò0°¸sÀV¹|Ð0ÑLÔLÐLØ'×-Ò-Ñ/Ô/×4Ò4°TÑ:Ô:�Fà—’˜TŸYšY vÑ.Ô.°Ð7GÔ1HÑH×PÒPÐUXÐPÑYÔYÑZÔZÐZØ—’˜DÑ!Ô!Ð!à#ð 	)Ø# IÔ.�GØ—L’L  w¤Ñ0Ô0×5Ò5°fÀAÈÈgÌÑDVÐ5ÑWÔWÐWØ˜YÔ'ð )à '¨$¡°Ò 2×6Ò6°t´zÑBÔB˜ØŸ	š	 $§)¢)¡+¤+×"4Ò"4¸Ð"4Ñ">Ô">Ñ?Ô?Ð?Ø!(¨4¡˜˜à!(˜à˜Ô(¨AÒ-Ð-Ø—K’K ¨u°^Ô/DÐ.DÀtÑ.K�KÑLÔLÐLà—’˜V˜GÑ$Ô$Ð$Ø”7�uœ}­e¬nÐ=Ð=Ð=Ø—G’G˜F‘O”O�OùñqX$ðt ˆs   ˜/¯3¶3)Nr	   r
   r   r   Nr   TFr   F)r   )N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   rg   r   ÚboolÚintr!   r(   Ústaticmethodr9   rD   rI   rY   rQ   Úno_gradr.   Ú__classcell__)r$   s   @r%   r   r      s¦  ø€ € € € € ðð ð6 #'ØØ#Ø$'Ø $Ø37Ø"%Ø$(Ø %Ø*,Ø!ð":ð ":àð":ð ˜”ð":ð ð	":ð
 ð":ð "ð":ð ð":ð ˜E %¨ ,Ô/Ô0ð":ð  ð":ð "ð":ð ð":ð %(ð":ð ð":ð ":ð ":ð ":ð ":ð ":ðH;ð ;ð ;ð ;ð ;ð ð!ð !ñ „\ð!ð ð*ð *ð *ñ „\ð*ð" ð8ð 8ñ „\ð8ð-ð -ð -ð €U„]�_„_ðeð eð eñ „_ðeð eð eð eð er&   r   )r~   r1   Útypingr   r   rQ   Ú_typesr   ÚoptimÚ	Optimizerr   r]   r&   r%   ú<module>rˆ      s�   ððð ð €€€Ø "Ð "Ð "Ð "Ð "Ð "Ð "Ð "à €€€à Ð Ð Ð Ð Ð ðPð Pð Pð Pð P�”Ô%ñ Pô Pð Pð Pð Pr&   