§
    ·¯¸j¤  ã            )       ó¨  — d Z ddlZddlZddlZddlZddlmZmZmZm	Z	m
Z
mZ ddlZ	 ddlmZ dZn# e$ r dZY nw xY wddlmZmZmZmZmZ dd	lmZ dd
lmZ ddlmZ  ej        e¦  «        ZdZdZdgg d¢g d¢g d¢dœZ ee!e
e"e"e"f         ee
e"e"e"f                  f         Z#dJdej$        de%dej$        fd„Z&de"de
e'df         de"fd„Z(edej)        dfdej$        de'dee
e"e"e"f                  de"de"d ej*        d!e%dej$        fd"„Z+	 dKd$ej,        d%e!de"fd&„Z-	 dKd$ej,        d%e!de
e"e%f         fd'„Z.	 	 	 dLdej$        d*e'd+e"d,e%dee%e
e%e!f         f         f
d-„Z/	 dMd/ej$        d0e!de
ej$        ej,        f         fd1„Z0dd2œd3eej$                 d4eej$                 d5eej$                 d6e"d7e"d8e"d9e%d:e'd;e#de"de"d%ee!         d<e!d=e%d!e%de%ddf"d>„Z1dd2œd3eej$                 d4eej$                 d5eej$                 d?eej$                 d@eej$                 d6e"d7e"d8e"d9e%dAe"d:e'd;e#de"de"d%ee!         d<e!d=e%d!e%de%ddf(dB„Z2dd2œd3eej$                 d4eej$                 d5eej$                 d6e"d7e"d8e"d9e%d:e'd;e#de"de"d%ee!         d<e!d=e%d!e%de%ddf"dC„Z3dd2œd3eej$                 d4eej$                 d5eej$                 d?eej$                 d@eej$                 d6e"d7e"d8e"d9e%dAe"d:e'd;e#de"de"d%ee!         d<e!d=e%d!e%de%ddf(dD„Z4 G dE„ dFej5        j6        ¦  «        Z7dGee!e	e"         e	e	e"                  f         dHee!e	e	e"                  f         dee
e"e"e"f                  fdI„Z8dS )Na¸   Muon Optimizer

Improved Muon optimizer implementation with flexible handling of high-dimensional tensors.

Combines PyTorch-style structure with options for:
- Batched spatial processing for convolutions in addition to flatten
- Optional spatial normalization
- Selectable coefficient presets
- Automatic fallback to AdamW for 1D / scalar parameters (biases, norms, etc.) and optional fallback via param groups
- AdaMuon (https://arxiv.org/abs/2507.11005)
- mUP eps damping factor (https://arxiv.org/abs/2512.05620v1)

TODO look into mUP LR scaling and independent weight-decay scale

Based on implementation by Keller Jordan, see
- https://github.com/KellerJordan/Muon/blob/master/muon.py
- https://github.com/KellerJordan/modded-nanogpt/blob/master/train_gpt.py
- https://github.com/KellerJordan/modded-nanogpt/blob/master/train_gpt_medium.py
- https://github.com/NoahAmsel/PolarExpress/blob/main/polar_express.py

Hacked together by Ross Wightman
é    N)ÚListÚMappingÚOptionalÚSequenceÚTupleÚUnion)ÚDTensorTFé   )Ú_add_scaled_Ú_check_capturable_devicesÚ
_get_valueÚ_validate_scalarÚ_zeros_scalar)ÚParamsT)Úadamw)ÚnadamwgH¯¼šò×z>é   )gu“VŽ@gš™™™™Àg�—nƒ@ @))gŸÍªÏÕV@g£¼”ÀgÑ"Ûù~j@)g´Èv¾Ÿš@ggÕçj+6ÀgF%u@)gåa¡Ö4ï@gÌH¿}]Àg’\þCúm@)g2æ®%ä@gÉå?$	Àgƒ/L¦
Fó?)g6«>W[±@gìQ¸…kÀg€H¿}8ó?))g°8Ó�y @gWŽr"b(7ÀgÓÙ=»9®0@)g‰3ç¬kT@gž)6ö$Àg—}U"Ïà?)g——Û")i@gVú}…ÖÀgêTÒ"†á?)gß¸%~Àb
@g_O"DveÀg)Ÿ™EÞß?)gÈÀ®çT`@gUß4Nž/ú¿gç³èè�Ù?)gÍ¶”˜§þ?gµ!ÙÌ²Áó¿güG‡ß†øÖ?)g··_äü³ý?g]ž“sió¿g??N!ÑÕÖ?)g9û+_ÿÿý?gŸW¾þÿó¿g—-¬|ýÿ×?))ghþ(ê'P @gÖ¾Í¹{6Àg«]!îÁ/@)gï=.Óõ+@g»íi¸©xÀgÛ’„l% à?)gúñ�º""@gPüõ+.ÀgPYJØ®1à?)gõ.¶69I
@gGË4àíñÀg—H5ºÝ?)gSO¥|g@gî•VÇù¿g@¡ýÄ™ˆØ?)gÎ�âXÏý?gžÀxDó¿gºPhæÕ?)gãkÐe€iý?gÂB?GÉØò¿gdÝˆÒØ¼Õ?)g1ïðýÿÿý?gÈj™ûÿÿó¿gZî¡öÿÿ×?)ÚoriginalÚquinticÚpolar_expressÚpolar_express_saferÚparamÚ
capturableÚreturnc                 ó4   — t          |r| j        nd ¬¦  «        S )N)Údevice)r   r   )r   r   s     ú_/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/optim/muon.pyÚ
_init_stepr   [   s   € Ý°
Ð D ¤ ÀÐEÑEÔEÐEó    ÚepsÚshape.c                 ó8   — |d         |d         }}| ||z  dz  z  S )u¨  Scale epsilon for Newton-Schulz based on matrix dimensions (Î¼P-style).

    For Î¼P compatibility, epsilon should scale as eps * sqrt(din/dout) to maintain
    consistent damping behavior across different model widths.

    Reference: https://arxiv.org/abs/2512.05620

    Args:
        eps: Base epsilon value
        shape: Shape of the matrix (out, in) or (batch, out, in)

    Returns:
        Scaled epsilon value
    éþÿÿÿéÿÿÿÿç      à?© )r    r!   ÚdoutÚdins       r   Úscale_eps_for_nsr)   _   s)   € ð( �r”˜E "œIˆ#€DØ�#˜‘* Ñ$Ñ$Ð$r   ç      ð?ÚGÚstepsÚcoefficientsÚsafety_factorÚdtypeÚ	scale_epsc           	      óp  — | j         dv sJ d| j         › d�¦   «         ‚t          |¦  «        }|dk    rt          |d         ¦  «        dk    sJ ‚||k    r
|d|…         n||d         g||z
  z  z   }|rt          || j        ¦  «        }|                      |d	¬
¦  «        }	|	                     d¦  «        |	                     d¦  «        k    }
|
r|	j        }	|rR|	                     |	                     ddd	¬¦  «         	                    |¦  «         
                    |¦  «        ¦  «         nR|	                     |	                     ddd	¬¦  «         	                    |¦  «                             |¬¦  «        ¦  «         t          ot          | t          ¦  «        }|r-|D ])\  }}}|	|	j        z  }||z  |||z  z  z   }||	z  ||	z  z   }	Œ*nê|	j         dk    rt          j        nt          j        }|	                     ¦   «         }	t          j        g |	j        dd…         ¢|	                     d¦  «        ‘R |	j        |	j        ¬¦  «        }t          j        |¦  «        }t          j        |	¦  «        }|D ]B\  }}} |||	|	j        dd|¬¦  «          |||||||¬¦  «          ||	||	|d|¬¦  «         ||	}}	ŒC|
r|	j        }	|	S )u”  Newton-Schulz quintic iteration to compute the zeroth power / orthogonalization of gradient.

    Supports batched operation over leading dimensions.

    See
    - https://github.com/KellerJordan/Muon/blob/master/muon.py
    - https://github.com/NoahAmsel/PolarExpress/blob/main/polar_express.py
    - https://github.com/KellerJordan/modded-nanogpt/blob/master/train_gpt.py

    Args:
        G: Input gradient tensor of shape (m, n) or (batch, m, n)
        steps: Number of Newton-Schulz iterations
        coefficients: Coefficients (a, b, c) for the iteration
        eps: Numerical stability epsilon for norm
        safety_factor: Multiplicative safety factor for norm (1.01 is common safety value in 'polar express' variants)
        dtype: Computation dtype
        scale_eps: If True, scale epsilon by sqrt(din/dout) for Î¼P compatibility

    Returns:
        Orthogonalized tensor of same shape as G
    )é   é   zInput must be 2D or 3D, got zD. Flatten batch dims first.r
   r   r3   Nr$   T)r/   Úcopyr#   r2   )r#   r$   )ÚdimÚkeepdim)Úmin)r   r/   g        r*   )ÚbetaÚalphaÚout)ÚndimÚlenr)   r!   ÚtoÚsizeÚmTÚdiv_ÚnormÚmulÚadd_Úclamp_Úhas_dtensorÚ
isinstancer	   ÚtorchÚbaddbmmÚaddmmÚ
contiguousÚemptyr   r/   Ú
empty_like)r+   r,   r-   r    r.   r/   r0   Únum_csÚcoeff_sequenceÚXÚ
transposedÚ
is_dtensorÚaÚbÚcÚAÚBÚmm_fnÚCs                      r   Úzeropower_via_newtonschulzrY   w   s  € ð< Œ6�VÐÐÐÐ`¸A¼FÐ`Ð`Ð`ÑÔÐÝ�ÑÔ€FØ�QŠ;ˆ;�3˜|¨AœÑ/Ô/°1Ò4Ð4Ð4Ð4à-2°fª_¨_�\ & 5 &Ô)Ð)Ø˜ RÔ(Ð)¨U°V©^Ñ<Ñ<ð ð ð -Ý˜s A¤GÑ,Ô,ˆà	�Š�5˜tˆÑ$Ô$€Að —’˜‘”˜aŸfšf R™jœjÒ(€JØð ØŒDˆð ð Yà	�Šˆq�vŠv�a˜X¨tˆvÑ4Ô4×8Ò8¸ÑGÔG×LÒLÈSÑQÔQÑRÔRÐRÐRà	�Šˆq�vŠv�a˜X¨tˆvÑ4Ô4×8Ò8¸ÑGÔG×NÒNÐSVÐNÑWÔWÑXÔXÐXåÐ7¥¨A­wÑ!7Ô!7€JØð à%ð 	 ð 	 ‰GˆAˆq�!Ø�A”D‘ˆAØ�A‘˜˜Q ™U™Ñ#ˆAØ�A‘˜˜Q™‘ˆAˆAð	 ð "#¤¨!¢ •”�µ´ˆð �LŠL‰NŒNˆÝŒKÐ3˜!œ' # 2 #œ,Ð3¨¯ª¨r©
¬
Ð3Ð3¸A¼HÈAÌGÐTÑTÔTˆÝÔ˜QÑÔˆÝÔ˜QÑÔˆð &ð 	ð 	‰GˆAˆq�!ð ˆE�!�Q˜œ 3¨c°qÐ9Ñ9Ô9Ð9ØˆE�!�Q˜ ¨°Ð2Ñ2Ô2Ð2ØˆE�!�Q˜ ¨°!Ð4Ñ4Ô4Ð4Ø�aˆqˆAˆAàð ØŒDˆà€Hr   Úmatch_rms_adamwÚparam_shapeÚadjust_lr_fnc                 óú   — t          | ¦  «        dk    r| d         | d         fnd\  }}|dk    rt          d||z  ¦  «        dz  S |dk    rdt          ||¦  «        dz  z  S |d	k    r||z  dz  S J d|› d�¦   «         ‚)av  Adjust learning rate based on parameter shape for Muon.

    Args:
        param_shape: Shape of the parameter tensor
        adjust_lr_fn: Scaling function name
            - "original": sqrt(max(1, out/in)) - Original Muon impl
            - "match_rms_adamw": 0.2 * sqrt(max(out, in)) - Kimi scaling
            - "rms_to_rms": sqrt(out/in) - Scion/Bernstein scaling
    r
   r#   r$   ©r*   r*   r   r%   rZ   çš™™™™™É?Ú
rms_to_rmsFúInvalid scaling function "z
" for Muon)r<   Úmax©r[   r\   Úout_chsÚin_chss       r   Úget_lr_scalerf   Ð   s´   € õ =@ÀÑ<LÔ<LÈqÒ<PÐ<P�{ 2”¨°B¬Ð8Ð8ÐV^�O€GˆVà�zÒ!Ð!å�1�g Ñ&Ñ'Ô'¨3Ñ.Ð.Ø	Ð*Ò	*Ð	*à•S˜ &Ñ)Ô)¨SÑ0Ñ0Ð0Ø	˜Ò	%Ð	%ð ˜&Ñ  SÑ(Ð(àKÐK°<ÐKÐKÐKÑKÔKˆur   c                 óì   — t          | ¦  «        dk    r| d         | d         fnd\  }}|dk    rdt          j        | ¦  «        dz  z  dfS |d	k    r
||z  dz  d
fS |dk    r|dz  d
fS J d|› d�¦   «         ‚)a!  Adjust learning rate based on parameter shape for AdaMuon.

    Args:
        param_shape: Shape of the update that went through Newton-Schulz, (out, in) or (spatial_prod, out, in)
        adjust_lr_fn: Scaling function name

    Returns:
        Tuple of (scale_factor, use_rms_norm)
    r
   r#   r$   r^   rZ   r_   r%   Tr`   FÚrsqrt_inç      à¿ra   z" for AdaMuon)r<   ÚmathÚprodrc   s       r   Úget_adamuon_lr_scalerl   í   s³   € õ =@ÀÑ<LÔ<LÈqÒ<PÐ<P�{ 2”¨°B¬Ð8Ð8ÐV^�O€GˆVàÐ(Ò(Ð(ð •T”Y˜{Ñ+Ô+¨sÑ2Ñ2°DÐ8Ð8Ø	˜Ò	%Ð	%Ø˜&Ñ  SÑ(¨%Ð/Ð/Ø	˜Ò	#Ð	#Ø˜‰~˜uÐ$Ð$àNÐN°<ÐNÐNÐNÑNÔNˆur   é   ç      `@Úmin_dim_sizeÚmax_aspect_ratioÚreturn_reasonc                 óœ  — | j         }| j        dk     st          d„ |D ¦   «         ¦  «        dk     r|rdndS |d         dk    s|d         dk    r|rdndS | j        dk    r!|d         }d}|dd	…         D ]}||z  }Œ||f}n|}t          |¦  «        }	|	|k     r|rdd
|	› �fS dS t	          |¦  «        }
|
|	z  }||k    r|rdd|d›�fS dS |rdndS )a†  Check if a parameter is suitable for Muon optimization.

    Args:
        param: Parameter tensor
        min_dim_size: Minimum size for non-unit dimensions
        max_aspect_ratio: Maximum allowed aspect ratio
        return_reason: If True, return (bool, reason_string), else just bool (faster)

    Returns:
        If return_reason=False: bool indicating suitability
        If return_reason=True: Tuple of (is_suitable, reason_string)

    Examples:
        (64, 128) -> True (or (True, "ok") if return_reason=True)
        (96, 3, 4, 4) -> True - will be flattened to (96, 48)
        (4, 2048) -> False - extreme aspect ratio
        (64,) -> False - insufficient dims
        (1, 196, 768) -> False - leading unit dims

    NOTE: these rules were created to balance complexity with covering common timm model cases
    Please let me know if there are non-optimal cases that you run into.
    r2   c              3   ó&   K  — | ]}|d k    ¯d V — ŒdS )r
   Nr&   )Ú.0Údim_sizes     r   ú	<genexpr>z(_is_suitable_for_muon.<locals>.<genexpr>(  s&   è è € ÐAÐA 8°H¸q²L°L˜Q°L°L°L°LÐAÐAr   )FÚinsufficient_dimsFr   r
   )FÚleading_unit_dimsr3   Nzmin_dim_too_small:zextreme_aspect_ratio:z.1f)TÚokT)r!   r;   Úsumr7   rb   )r   ro   rp   rq   ÚsÚout_chÚin_ch_with_spatialÚdÚ
check_dimsÚmin_sizeÚmax_sizeÚaspect_ratios               r   Ú_is_suitable_for_muonrƒ   	  sY  € ð: 	Œ€Aà„z�A‚~€~�ÐAÐA¨qÐAÑAÔAÑAÔAÀAÒEÐEØ/<ÐGÐ+Ð+À%ÐGð 	ˆ„tˆq‚y€y�A�a”D˜A’I�IØ/<ÐGÐ+Ð+À%ÐGà„z�Q‚€ð �1”ˆØÐØ�1�2�2”ð 	$ð 	$ˆAØ !Ñ#ÐÐØÐ0Ð1ˆ
ˆ
ð ˆ
õ �:‰Œ€HØ�,ÒÐØð 	:ØÐ9¨xÐ9Ð9Ð9Ð9Øˆuõ �:‰Œ€HØ˜hÑ&€LØÐ&Ò&Ð&Øð 	EØÐD°,ÐDÐDÐDÐDÐDØˆuà(Ð2ˆ<ˆ<¨dÐ2r   ÚflattenÚtensorÚmodec                 óf  — | j         }| j        dk    r| |fS | j        dk     rt          d| j        › �¦  «        ‚| j         dd…         \  }}|dk    r|                      |d¦  «        |fS |dk    r2|                      ||d¦  «        }|                     ddd¦  «        }||fS t          d	|› �¦  «        ‚)
a´  Reshape high-dimensional tensor for Muon processing.

    Args:
        tensor: Input tensor of shape (out, in, *spatial)
        mode: How to handle spatial dimensions
            - "flatten": Flatten spatial into output dimension (out, in*H*W)
            - "batched": Batch over spatial positions (spatial_prod, out, in) for per-position orthogonalization

    Returns:
        Reshaped tensor and original shape for restoration
    r2   z,Tensor must have at least 2 dimensions, got Nr„   r$   Úbatchedr   r
   zUnknown mode: )r!   r;   Ú
ValueErrorÚreshapeÚpermute)r…   r†   Úoriginal_shaper|   Úin_chÚreshapeds         r   Úreshape_for_muonr�   Q  sÚ   € ð ”\€NØ„{�aÒÐØ�~Ð%Ð%Ø„{�Q‚€ÝÐUÈÌÐUÐUÑVÔVÐVà”L  ! Ô$�M€FˆEØˆyÒÐà�~Š~˜f bÑ)Ô)¨>Ð9Ð9Ø	�Ò	Ð	ð —>’> &¨%°Ñ4Ô4ˆØ×#Ò# A q¨!Ñ,Ô,ˆØ˜Ð'Ð'åÐ0¨$Ð0Ð0Ñ1Ô1Ð1r   )r   ÚparamsÚgradsÚmomentum_bufsÚlrÚweight_decayÚmomentumÚnesterovÚns_stepsÚns_coefficientsÚ	conv_modeÚnormalize_spatialc                óD   — t          | |||||||||	|
|||||¬¦  «         dS )z8Functional API that performs Muon algorithm computation.©r“   r”   r•   r–   r—   r˜   r    r.   r\   r™   rš   r0   r   N)Ú_single_tensor_muon)r�   r‘   r’   r“   r”   r•   r–   r—   r˜   r    r.   r\   r™   rš   r0   r   s                   r   Úmuonrž   t  sU   € õ( ØØØØØ!ØØØØ'ØØ#Ø!ØØ+ØØð!ñ ô ð ð ð r   Úexp_avg_sqsÚstate_stepsÚbeta2c                óB   — t          | ||||f|||||	|
||||||||dœŽ dS )a<  Functional API that performs AdaMuon algorithm computation.

    AdaMuon extends Muon with element-wise second moment estimation applied
    to orthogonalized update directions, providing Adam-like adaptive scaling
    while preserving Muon's geometric benefits.

    Reference: https://arxiv.org/abs/2507.11005
    ©r“   r”   r•   r–   r¡   r—   r˜   r    r.   r\   r™   rš   r0   r   N)Ú_single_tensor_adamuon)r�   r‘   r’   rŸ   r    r“   r”   r•   r–   r¡   r—   r˜   r    r.   r\   r™   rš   r0   r   s                      r   Úadamuonr¥   œ  se   € õ< ØØØØØðð Ø!ØØØØØ'ØØ#Ø!ØØ+ØØð'ð ð ð ð ð r   c          	      ó¼  — t          |t          ¦  «        }t          | ¦  «        D �]5\  }}||         }||         }|                     d||z  z
  ¦  «         |                     |d|z
  ¦  «         |r|                     ||¦  «        n|                     ¦   «         }|j        dk    rt          ||¬¦  «        \  }}n	|}|j        }t          ||||	|
|¬¦  «        }|rt          |j        |¦  «        }nd}|dk    r7|j        dk    r,|r||j        d         dz  z  }|                     dd	d¦  «        }|                     |¦  «        }t          ||| |z  ¦  «         �Œ7d
S )zSingle tensor Muon update.r
   r*   r3   ©r†   ©r    r.   r0   rˆ   r   ri   r2   N)Úresolve_ns_coefficientsÚ_COEFFICIENTSÚ	enumerateÚmul_Úlerp_Úcloner;   r�   r!   rY   rf   r‹   rŠ   r   )r�   r‘   r’   r“   r”   r•   r–   r—   r˜   r    r.   r\   r™   rš   r0   r   Úir   ÚgradÚmomentum_bufÚupdateÚupdate_reshapedrŒ   Úupdate_orthoÚscales                            r   r�   r�   Ñ  s°  € õ( .¨o½}ÑMÔM€Oå˜fÑ%Ô%ð -7ñ -7‰ˆˆ5Ø�QŒxˆØ$ QÔ'ˆð 	�
Š
�1�r˜LÑ(Ñ(Ñ)Ô)Ð)ð 	×Ò˜4  h¡Ñ/Ô/Ð/Ø7?ÐY�—’˜L¨(Ñ3Ô3Ð3À\×EWÒEWÑEYÔEYˆð Œ;˜!ÒÐÝ.>¸vÈIÐ.VÑ.VÔ.VÑ+ˆO˜^˜^à$ˆOØ#œ\ˆNõ 2ØØØØØ'Øð
ñ 
ô 
ˆð ð 	Ý  Ô!3°\ÑBÔBˆEˆEàˆEð ˜	Ò!Ð! lÔ&7¸1Ò&<Ð&<Ø ð 7Ø˜Ô+¨AÔ.°$Ñ6Ñ6�à'×/Ò/°°1°aÑ8Ô8ˆLð $×+Ò+¨NÑ;Ô;ˆõ 	�U˜L¨2¨#°©+Ñ6Ô6Ð6Ñ6ð[-7ð -7r   c          	      ór  — t          |t          ¦  «        }|rt          | |¦  «         t          | ¦  «        D �]~\  }}||         }||         }||         }||         }|                     d¦  «         |r|nt          |¦  «        }|                     d||z  z
  ¦  «         |                     |d|z
  ¦  «         |r|                     ||¦  «        n|                     ¦   «         }|j	        dk    rt          ||¬¦  «        \  }}n	|}|j        }t          ||
||||¬¦  «        }|rt          |j        |¦  «        \  }}nd\  }}|dk    r"|j	        dk    r|                     ddd	¦  «        }|                     |¦  «        }|                     |	¦  «                             ||d|	z
  ¬
¦  «         |r(|                     ¦   «                              |¦  «        } n^t%          j        |¦  «        rdt%          j        |	|¦  «        z
  nd|	|z  z
  }!||!z                       ¦   «                              |¦  «        } || z  }"|r,|"                     ¦   «                              |¦  «        }#|"|#z  }"|dk    r1t-          |¦  «        dk    r|rd}$|dd…         D ]}%|$|%z  }$Œ||$dz  z  }t/          ||"| |z  ¦  «         �Œ€dS )u)  Single tensor AdaMuon update.

    AdaMuon applies second-moment estimation to the orthogonalized directions,
    then rescales using RMS-alignment to maintain stable step sizes.

    Algorithm:
        1. Update momentum buffer: M = Î²â‚�Â·M + (1-Î²â‚�)Â·G
        2. Orthogonalize: O = Newton-Schulz(M) or Newton-Schulz(nesterov_update)
        3. Update second moment: v = Î²â‚‚Â·v + (1-Î²â‚‚)Â·OÂ²
        4. Bias correct: vÌ‚ = v/(1-Î²â‚‚^t)
        5. Adaptive scaling: Ã” = O / (âˆšvÌ‚ + Îµ)
        6. RMS-aligned rescaling and apply update
    r
   r*   r3   r§   r¨   )r*   Frˆ   r2   r   )ÚvalueNri   )r©   rª   r   r«   rC   r   r¬   r­   r®   r;   r�   r!   rY   rl   r‹   rŠ   Úaddcmul_ÚsqrtrG   Ú	is_tensorÚpowrA   r<   r   )&r�   r‘   r’   rŸ   r    r“   r”   r•   r–   r¡   r—   r˜   r    r.   r\   r™   rš   r0   r   r¯   r   r°   r±   Ú
exp_avg_sqÚstep_tÚstepr²   r³   rŒ   r´   rµ   Úuse_rms_normÚdenomÚbias_correction2Úupdate_adaptiveÚupdate_normÚspatial_prodr~   s&                                         r   r¤   r¤     sD  € õF .¨o½}ÑMÔM€OØð 7Ý! &¨+Ñ6Ô6Ð6å˜fÑ%Ô%ð M:ñ M:‰ˆˆ5Ø�QŒxˆØ$ QÔ'ˆØ  ”^ˆ
Ø˜Q”ˆð 	�Š�A‰ŒˆØ#Ð;ˆvˆv­°FÑ);Ô);ˆð 	�
Š
�1�r˜LÑ(Ñ(Ñ)Ô)Ð)ð 	×Ò˜4  h¡Ñ/Ô/Ð/Ø7?ÐY�—’˜L¨(Ñ3Ô3Ð3À\×EWÒEWÑEYÔEYˆð Œ;˜!ÒÐÝ.>¸vÈIÐ.VÑ.VÔ.VÑ+ˆO˜^˜^à$ˆOØ#œ\ˆNõ 2ØØØØØ'Øð
ñ 
ô 
ˆð ð 	-Ý"6°|Ô7IÈ<Ñ"XÔ"XÑˆE�<�<à",ÑˆE�<ð ˜	Ò!Ð! lÔ&7¸1Ò&<Ð&<à'×/Ò/°°1°aÑ8Ô8ˆLØ#×+Ò+¨NÑ;Ô;ˆð 	�Š˜ÑÔ×'Ò'¨°lÈ#ÐPUÉ+Ð'ÑVÔVÐVàð 	Eà—O’OÑ%Ô%×*Ò*¨3Ñ/Ô/ˆEˆEõ @E¼ÈtÑ?TÔ?TÐm˜s¥U¤Y¨u°dÑ%;Ô%;Ñ;Ð;ÐZ]Ð`eÐimÑ`mÑZmÐØÐ"2Ñ2×8Ò8Ñ:Ô:×?Ò?ÀÑDÔDˆEð '¨Ñ.ˆð ð 	<à)×.Ò.Ñ0Ô0×5Ò5°cÑ:Ô:ˆKØ-°Ñ;ˆOð ˜	Ò!Ð!¥c¨.Ñ&9Ô&9¸QÒ&>Ð&>Ø ð .Ø �Ø'¨¨¨Ô+ð &ð &�AØ  AÑ%�L�LØ˜¨Ñ-Ñ-�õ 	�U˜O¨b¨S°5©[Ñ9Ô9Ð9Ñ9ð[M:ð M:r   c            '       óú   ‡ — e Zd ZdZddddededdd	d
dddddddfdededededede	de
dededee         dededee         dedeeef         dededed ef&ˆ fd!„Zˆ fd"„Z ej        ¦   «         d$d#„¦   «         Zˆ xZS )%ÚMuonaþ  Muon - MomentUm Orthogonalized by Newton-schulz

    Combines Muon for 2D+ parameters (weight matrices) with AdamW for 1D parameters (biases, norms) and
    parameter groups with 'use_fallback=True' set (or 'use_muon=False' for compatibility).

    Supports two algorithms:
    - "muon": Standard Muon algorithm with momentum + orthogonalization
    - "adamuon": AdaMuon algorithm that adds element-wise second moment estimation
                 to orthogonalized directions for Adam-like adaptive scaling
    g{®Gáz”?r   çffffffî?Fr   r*   rZ   r„   TN)gÍÌÌÌÌÌì?rÇ   rž   r�   r“   r”   r•   r–   r—   r˜   r    r.   r\   r™   rš   Úadamw_lrÚfallback_lr_scaleÚbetasÚalgor0   r   Úverbosec                 óZ  •— t          d|¦  «         t          d|¦  «         t          d|d¬¦  «         t          d|¦  «         |dvrt          d|› �¦  «        ‚|d	vrt          d
|› d�¦  «        ‚|�Yt          j        dt          d¬¦  «         t          j        |¦  «        rt          d¦  «        ‚|dk    rt          d¦  «        ‚||z  }t          d"i d|“d|“d|“d|“d|“d|“d|“d|	“d|
“d|“d|“d|“d|“d|“d|“d |“d!|“Ž}t          ¦   «          	                    ||¦  «         dS )#u0
   Create Muon optimizer.
        Args:
            params: Iterable of parameters or dicts defining parameter groups
            lr: Learning rate (default: 0.02 for Muon parameters)
            weight_decay: Weight decay coefficient
            momentum: Momentum factor for Muon
            nesterov: Whether to use Nesterov momentum
            ns_steps: Number of Newton-Schulz iterations
            ns_coefficients: Coefficients for NS iteration
            eps: Numerical stability epsilon
            safety_factor: Multiplicative safety factor for NS norm
            adjust_lr_fn: LR adjustment function - "original", "match_rms_adamw", or "rms_to_rms".
                For adamuon mode, can set to None to disable (RMS rescaling handles scaling).
            conv_mode: How to handle convolutions - "flatten" or "batched"
            normalize_spatial: Whether to normalize by sqrt(spatial_size) in batched mode
            fallback_lr_scale: Scale factor applied to lr for AdamW fallback parameters.
                The effective fallback LR is lr * fallback_lr_scale. This ensures the LR scheduler
                properly schedules fallback parameters.
            adamw_lr: Deprecated. Use fallback_lr_scale instead.
            betas: Beta coefficients - (beta1, beta2) where beta1 is used for AdamW fallback
                and beta2 is used for both AdamW fallback and AdaMuon second moment
            algo: Algorithm - "muon" for standard Muon, "adamuon" for AdaMuon with
                adaptive second moment estimation (https://arxiv.org/abs/2507.11005)
            scale_eps: If True, scale epsilon by sqrt(din/dout) in Newton-Schulz for Î¼P
                compatibility (https://arxiv.org/abs/2512.05620)
            capturable: Whether this instance is safe to capture in a CUDA graph. Capturable mode supports
                tensor learning rates and requires optimizer state tensors to live on the parameter device.
            verbose: Log parameter routing decisions (Muon vs AdamW)

        Example:
            ```python
            # Simple usage - automatically uses Muon for 2D+ params, AdamW for 1D
            optimizer = Muon(model.parameters(), lr=0.02)

            # Use AdaMuon algorithm for adaptive scaling
            optimizer = Muon(model.parameters(), lr=6e-4, algo="adamuon")

            # Manual control over parameter groups
            optimizer = Muon([
                {'params': weight_matrices, 'lr': 0.02},
                {'params': biases, 'use_fallback': True, 'lr': 3e-4}, # use AdamW if use_fallback=True
            ])
            ```
        zlearning rater”   r•   r*   )Ú	max_valueÚepsilon)r„   rˆ   zInvalid conv_mode: )rž   r¥   zInvalid algo: z. Must be 'muon' or 'adamuon'Nzpadamw_lr is deprecated, use fallback_lr_scale=adamw_lr/lr instead. adamw_lr will be removed in a future release.r2   )Ú
stacklevelzHadamw_lr is not supported with tensor lr; use fallback_lr_scale instead.r   z8Cannot compute fallback_lr_scale from adamw_lr when lr=0r“   r–   r—   r˜   r    r.   r\   r™   rš   rÉ   rÊ   rË   r0   r   rÌ   r&   )
r   r‰   ÚwarningsÚwarnÚFutureWarningrG   rº   ÚdictÚsuperÚ__init__)Úselfr�   r“   r”   r•   r–   r—   r˜   r    r.   r\   r™   rš   rÈ   rÉ   rÊ   rË   r0   r   rÌ   ÚdefaultsÚ	__class__s                        €r   rÖ   zMuon.__init__š  s  ø€ õD 	˜¨"Ñ-Ô-Ð-Ý˜¨Ñ6Ô6Ð6Ý˜ X¸Ð=Ñ=Ô=Ð=Ý˜ CÑ(Ô(Ð(ØÐ2Ð2Ð2ÝÐ>°9Ð>Ð>Ñ?Ô?Ð?ØÐ*Ð*Ð*ÝÐQ¨dÐQÐQÐQÑRÔRÐRàÐÝŒMð@åØð	ñ ô ð õ Œ˜rÑ"Ô"ð mÝ Ð!kÑlÔlÐlØ�QŠwˆwÝ Ð![Ñ\Ô\Ð\Ø (¨2¡Ðåð 
ð 
ð 
Øˆrð
à%˜ð
ð �Xð
ð �Xð	
ð
 �Xð
ð ,˜Oð
ð �ð
ð (˜-ð
ð &˜ð
ð  �ið
ð 0Ð/ð
ð 0Ð/ð
ð �%ð
ð �ð
ð  �ið
ð  "�zð!
ð" �Gð#
ˆõ& 	‰Œ×Ò˜ Ñ*Ô*Ð*Ð*Ð*r   c                 óZ  •— t          ¦   «                              |¦  «         | j        D ]€}|                     dd¦  «         |                     dd¦  «         |                     dd¦  «         d|vr8|                     d|d         ¦  «        }|d         d	k    r||d         z  nd
|d<   Œ�d S )NrË   rž   r0   Fr   rÉ   rÈ   r“   r   r*   )rÕ   Ú__setstate__Úparam_groupsÚ
setdefaultÚpop)r×   ÚstateÚgrouprÈ   rÙ   s       €r   rÛ   zMuon.__setstate__  sÉ   ø€ Ý‰Œ×Ò˜UÑ#Ô#Ð#ØÔ&ð 	að 	aˆEØ×Ò˜V VÑ,Ô,Ð,Ø×Ò˜[¨%Ñ0Ô0Ð0Ø×Ò˜\¨5Ñ1Ô1Ð1à"¨%Ð/Ð/Ø Ÿ9š9 Z°°t´Ñ=Ô=�ØGLÈTÄ{ÐVWÒGWÐGW¨X¸¸d¼Ñ-CÐ-CÐ]`�Ð)Ñ*øð	að 	ar   c                 óf  — | j                              dd¦  «        rIt          | d¦  «        r|                      ¦   «          n$t          | d¦  «        r|                      ¦   «          d}|�5t          j        ¦   «         5   |¦   «         }ddd¦  «         n# 1 swxY w Y   | j                              dd¦  «        }d}d}|ri nd}| j        D �]z}|                     dd	¦  «        }g }	g }
g }g }g }g }g }g }g }g }|d
         D �]Ì}|j        €Œ|j        j	        rt          d¦  «        ‚| j        |         }d|vr¯d}|                     dd¦  «        r
d|d<   |rd}nAd|v r|d         |d<   |rd}n-|rt          |d¬¦  «        \  }}nt          |d¬¦  «        }||d<   |�J|�Hd                     d„ |j        D ¦   «         ¦  «        }||vrg ||<   ||                              |¦  «         |d         }|rô|	                     |¦  «         |
                     |j        ¦  «         |dz  }d|vr#t          j        |t
          j        ¬¦  «        |d<   |                     |d         ¦  «         |dk    rvd|vr<t%          ||d         ¦  «        |d<   t          j        |t
          j        ¬¦  «        |d<   |                     |d         ¦  «         |                     |d         ¦  «         �Œä|                     |¦  «         |                     |j        ¦  «         |dz  }d|vr_t%          ||d         ¦  «        |d<   t          j        |t
          j        ¬¦  «        |d<   t          j        |t
          j        ¬¦  «        |d<   |                     |d         ¦  «         |                     |d         ¦  «         |                     |d         ¦  «         �ŒÎ|	rë|dk    rx|d         \  }}t'          |	|
|||f|d         |d         |d         |d         ||d          |d!         |d"         |d#         |d$         |d%         |d&         |d'         |d         d(œŽ nmt)          |	|
||d         |d         |d         |d         |d          |d!         |d"         |d#         |d$         |d%         |d&         |d'         |d         ¬)¦  «         |rˆ|d         \  }}|d         |d*         z  }|d         r2t+          |||||d||||d         |d"         dd|d         d¬+¦  «         �ŒHt-          ||||g |dd||||d         |d"         dd|d         d¬,¦  «         �Œ||�rèt/          |¦  «        dk    �rÔt0                               d-|› d.|› d/�¦  «         i }t5          |                     ¦   «         ¦  «        D ].\  }}|D ]&}||vrg ||<   ||                              |¦  «         Œ'Œ/g }t5          |                     ¦   «         ¦  «        D ],\  }} |                     |› d0t/          | ¦  «        › �¦  «         Œ-t0                               d1d2                     |¦  «        › �¦  «         t0                               t:          j        ¦  «        r½t5          |                     ¦   «         ¦  «        D ]›\  }} |d3k    rd4nd5}!t0                               d6|› d7|!› d8�¦  «         | dd9…         D ]}"t0                               d:|"› �¦  «         Œ t/          | ¦  «        d9k    r.t0                               d;t/          | ¦  «        d9z
  › d<�¦  «         Œœ|S )=z$Performs a single optimization step.r   FÚ'_accelerator_graph_capture_health_checkÚ _cuda_graph_capture_health_checkNrÌ   r   rË   rž   r�   z&Muon does not support sparse gradientsÚuse_muonÚuse_fallbackÚuse_fallback_flagÚuse_muon_flagT)rq   Úxc              3   ó4   K  — | ]}t          |¦  «        V — Œd S ©N)Ústr)rt   r{   s     r   rv   zMuon.step.<locals>.<genexpr>Y  s(   è è € Ð,EÐ,E¸­S°©V¬VÐ,EÐ,EÐ,EÐ,EÐ,EÐ,Er   r
   Úmomentum_buffer)Úmemory_formatr¥   r¾   r¼   Úexp_avgrÊ   r“   r”   r•   r–   r—   r˜   r    r.   r\   r™   rš   r0   r£   rœ   rÉ   )
ÚforeachÚbeta1r¡   r“   r”   r    ÚcautionÚmaximizer   Úmax_lr)rï   Úamsgradrð   r¡   r“   r”   r    rñ   rò   r   ró   zMuon parameter routing: z Muon, z AdamWú=z  Breakdown: ú, ry   rÆ   ÚAdamWz    z -> ú:é
   z      z      ... and z more)rØ   ÚgetÚhasattrrâ   rã   rG   Úenable_gradrÜ   r°   Ú	is_sparseÚRuntimeErrorrß   rƒ   Újoinr!   ÚappendÚ
zeros_likeÚpreserve_formatr   r¥   rž   r   r   r<   Ú_loggerÚinfoÚsortedÚitemsÚisEnabledForÚloggingÚINFO)#r×   ÚclosureÚlossrÌ   Ú
muon_countÚadamw_countÚrouting_reasonsrà   rË   Úmuon_paramsÚ
muon_gradsÚmuon_momentum_bufsÚmuon_exp_avg_sqsÚmuon_state_stepsÚadamw_paramsÚadamw_gradsÚadamw_exp_avgsÚadamw_exp_avg_sqsÚadamw_state_stepsÚprß   ÚreasonÚsuitableÚ	shape_strrä   Ú_r¡   rð   Úfallback_lrÚreason_groupsÚreasonsÚreason_summaryÚshapesÚoptimizer_namer!   s#                                      r   r¾   z	Muon.step  s	  € ð Œ=×Ò˜\¨5Ñ1Ô1ð 	8Ý�tÐFÑGÔGð 8Ø×<Ò<Ñ>Ô>Ð>Ð>Ý˜ÐAÑBÔBð 8Ø×5Ò5Ñ7Ô7Ð7àˆØÐÝÔ"Ñ$Ô$ð !ð !Ø�w‘y”y�ð!ð !ð !ñ !ô !ð !ð !ð !ð !ð !ð !øøøð !ð !ð !ð !ð ”-×#Ò# I¨uÑ5Ô5ˆð ˆ
ØˆØ 'Ð1˜"˜"¨TˆàÔ&ð r	ñ r	ˆEØ—9’9˜V VÑ,Ô,ˆDð ˆKØˆJØ!#Ðà!ÐØ!ÐàˆLØˆKØˆNØ "ÐØ "Ðà˜8”_ð H<ñ H<�Ø”6�>Øà”6Ô#ð QÝ&Ð'OÑPÔPÐPàœ
 1œ�ð  UÐ*Ð*à!�FØ—y’y °Ñ7Ô7ð 5à,1˜˜jÑ)Ø"ð 9Ø%8˜FøØ# uÐ,Ð,à,1°*Ô,=˜˜jÑ)Ø"ð 5Ø%4˜Føð #ð UÝ/DÀQÐVZÐ/[Ñ/[Ô/[Ñ,˜H f få'<¸QÈeÐ'TÑ'TÔ'T˜HØ,4˜˜jÑ)ð 'Ð2°vÐ7IØ$'§H¢HÐ,EÐ,E¸Q¼WÐ,EÑ,EÔ,EÑ$EÔ$E˜	Ø$¨OÐ;Ð;Ø9;˜O¨IÑ6Ø'¨	Ô2×9Ò9¸&ÑAÔAÐAð ! Ô,�Øð  <à×&Ò& qÑ)Ô)Ð)Ø×%Ò% a¤fÑ-Ô-Ð-Ø !‘O�Jð )°Ð5Ð5Ý38Ô3CÀAÕUZÔUjÐ3kÑ3kÔ3k˜Ð/Ñ0Ø&×-Ò-¨eÐ4EÔ.FÑGÔGÐGð ˜yÒ(Ð(Ø!¨Ð.Ð.Ý,6°q¸%ÀÔ:MÑ,NÔ,N˜E &™MÝ27Ô2BÀ1ÕTYÔTiÐ2jÑ2jÔ2j˜E ,Ñ/Ø(×/Ò/°°lÔ0CÑDÔDÐDØ(×/Ò/°°f´Ñ>Ô>Ð>ùð !×'Ò'¨Ñ*Ô*Ð*Ø×&Ò& q¤vÑ.Ô.Ð.Ø 1Ñ$�Kð  UÐ*Ð*Ý(2°1°e¸LÔ6IÑ(JÔ(J˜˜f™Ý+0Ô+;¸AÍUÔMbÐ+cÑ+cÔ+c˜˜iÑ(Ý.3Ô.>¸qÕPUÔPeÐ.fÑ.fÔ.f˜˜lÑ+à"×)Ò)¨%°	Ô*:Ñ;Ô;Ð;Ø%×,Ò,¨U°<Ô-@ÑAÔAÐAØ%×,Ò,¨U°6¬]Ñ;Ô;Ð;Ñ;ð ð *Ø˜9Ò$Ð$Ø$ Wœ~‘H�A�uÝØ#Ø"Ø*Ø(Ø(ðð ! œ;Ø%*¨>Ô%:Ø!& zÔ!2Ø!& zÔ!2Ø#Ø!& zÔ!2Ø(-Ð.?Ô(@Ø! %œLØ&+¨OÔ&<Ø%*¨>Ô%:Ø"'¨Ô"4Ø*/Ð0CÔ*DØ"'¨Ô"4Ø#(¨Ô#6ð'ð ð ð ð õ, Ø#Ø"Ø*Ø  œ;Ø%*¨>Ô%:Ø!& zÔ!2Ø!& zÔ!2Ø!& zÔ!2Ø(-Ð.?Ô(@Ø! %œLØ&+¨OÔ&<Ø%*¨>Ô%:Ø"'¨Ô"4Ø*/Ð0CÔ*DØ"'¨Ô"4Ø#(¨Ô#6ð!ñ ô ð ð( ð )Ø$ Wœ~‘��uØ# Dœk¨EÐ2EÔ,FÑF�Ø˜Ô$ð &åØ$Ø#Ø&Ø)Ø)Ø $Ø#Ø#Ø&Ø%*¨>Ô%:Ø! %œLØ %Ø!&Ø#(¨Ô#6Ø#ðñ ô ð ñ õ$ Ø$Ø#Ø&Ø)ØØ)Ø $Ø %Ø#Ø#Ø&Ø%*¨>Ô%:Ø! %œLØ %Ø!&Ø#(¨Ô#6Ø#ð#ñ ô ð ùð* ñ 	O�s ?Ñ3Ô3°aÒ7Ñ7å�LŠLÐZ°JÐZÐZÀ{ÐZÐZÐZÑ[Ô[Ð[ð ˆMÝ&,¨_×-BÒ-BÑ-DÔ-DÑ&EÔ&Eð <ð <Ñ"�	˜7Ø%ð <ð <�FØ ]Ð2Ð2Ø02˜ fÑ-Ø! &Ô)×0Ò0°Ñ;Ô;Ð;Ð;ð<ð  ˆNÝ"(¨×)<Ò)<Ñ)>Ô)>Ñ"?Ô"?ð Að A‘�˜Ø×%Ò%¨Ð&?Ð&?µ#°f±+´+Ð&?Ð&?Ñ@Ô@Ð@Ð@Ý�LŠLÐD¨¯ª°>Ñ)BÔ)BÐDÐDÑEÔEÐEõ ×#Ò#¥G¤LÑ1Ô1ð OÝ&,¨]×-@Ò-@Ñ-BÔ-BÑ&CÔ&Cð Oð O‘N�F˜FØ/5¸ª~¨~ V VÀ7�NÝ—L’LÐ!E¨Ð!EÐ!E°NÐ!EÐ!EÐ!EÑFÔFÐFØ!'¨¨¨¤ð 7ð 7˜ÝŸšÐ%5¨eÐ%5Ð%5Ñ6Ô6Ð6Ð6Ý˜6‘{”{ RÒ'Ð'ÝŸšÐ%Mµc¸&±k´kÀBÑ6FÐ%MÐ%MÐ%MÑNÔNÐNøàˆs   Á<BÂBÂBrê   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚDEFAULT_NS_STEPSÚMUON_EPSr   ÚfloatÚboolÚintÚNSCoeffr   rë   r   rÖ   rÛ   rG   Úno_gradr¾   Ú__classcell__)rÙ   s   @r   rÆ   rÆ   Ž  sº  ø€ € € € € ð	ð 	ð Ø"#Ø"Ø"Ø,Ø'0Ø!Ø#&Ø*;Ø&Ø&*Ø(,Ø'*Ø)4ØØ#Ø$Ø!ð)k+ð k+àðk+ð ðk+ð  ð	k+ð
 ðk+ð ðk+ð ðk+ð %ðk+ð ðk+ð !ðk+ð # 3œ-ðk+ð ðk+ð  $ðk+ð ˜u”oðk+ð  %ðk+ð  ˜ ˜Ô&ð!k+ð" ð#k+ð$ ð%k+ð& ð'k+ð( ð)k+ð k+ð k+ð k+ð k+ð k+ðZ	að 	að 	að 	að 	að €U„]�_„_ðeð eð eñ „_ðeð eð eð eð er   rÆ   r·   Úpresetsc                 óN  ‡‡‡	— d„ Š	d„ Šdt           t                   dt          t          t          t          f         fˆˆ	fd„Št          | t          ¦  «        r”| |vrId                     t          |                     ¦   «         ¦  «        ¦  «        }t          d| › d|› �¦  «        ‚||          } ‰	|¦  «        rt          |¦  «        d	k    rt          d
| › d�¦  «        ‚ˆfd„|D ¦   «         S  ‰	| ¦  «        st          d¦  «        ‚t          | ¦  «        dk    r't          ˆfd„| D ¦   «         ¦  «        r ‰| ¦  «        gS g }t          | ¦  «        D ]C\  }} ‰	|¦  «        st          d|› d|›�¦  «        ‚|                      ‰|¦  «        ¦  «         ŒD|st          d¦  «        ‚|S )Nc                 óf   — t          | t          ¦  «        ot          | t          t          f¦  «         S rê   )rF   r   rë   Úbytes©rè   s    r   ú<lambda>z)resolve_ns_coefficients.<locals>.<lambda>   s&   € •z !¥XÑ.Ô.ÐRµzÀ!ÅcÍ5À\Ñ7RÔ7RÐ3R€ r   c                 ób   — t          | t          j        ¦  «        ot          | t          ¦  «         S rê   )rF   ÚnumbersÚRealr+  r4  s    r   r5  z)resolve_ns_coefficients.<locals>.<lambda>  s$   € �
 1¥g¤lÑ3Ô3ÐO½JÀqÍ$Ñ<OÔ<OÐ8O€ r   rè   r   c                 óþ   •—  ‰| ¦  «        r.t          | ¦  «        dk    st          ˆfd„| D ¦   «         ¦  «        st          d| ›�¦  «        ‚| \  }}}t          |¦  «        t          |¦  «        t          |¦  «        fS )Nr3   c              3   ó.   •K  — | ]} ‰|¦  «        V — Œd S rê   r&   ©rt   ÚvÚis_reals     €r   rv   z<resolve_ns_coefficients.<locals>.as_coeff.<locals>.<genexpr>  s+   øè è € Ð2IÐ2IÀ!°7°7¸1±:´:Ð2IÐ2IÐ2IÐ2IÐ2IÐ2Ir   z3Coefficient must be length-3 of real numbers, got: )r<   Úallr‰   r*  )rè   rR   rS   rT   r=  Úis_seqs       €€r   Úas_coeffz)resolve_ns_coefficients.<locals>.as_coeff  s…   ø€ Øˆv�a‰yŒyð 	Z�C ™FœF ašK˜K­sÐ2IÐ2IÐ2IÐ2IÀqÐ2IÑ2IÔ2IÑ/IÔ/I˜KÝÐXÐSTÐXÐXÑYÔYÐYØ‰ˆˆ1ˆaÝ�Q‰xŒx�˜q™œ¥5¨¡8¤8Ð+Ð+r   rö   zUnknown coefficients preset 'z'. Valid options: r   zPreset 'z' is empty or invalidc                 ó&   •— g | ]} ‰|¦  «        ‘ŒS r&   r&   )rt   Úitemr@  s     €r   ú
<listcomp>z+resolve_ns_coefficients.<locals>.<listcomp>  s!   ø€ Ð/Ð/Ð/ 4��˜‘”Ð/Ð/Ð/r   z]Coefficients must be a preset name (str), a 3-sequence (a,b,c), or a sequence of 3-sequences.r3   c              3   ó.   •K  — | ]} ‰|¦  «        V — Œd S rê   r&   r;  s     €r   rv   z*resolve_ns_coefficients.<locals>.<genexpr>  s+   øè è € Ð9Ð9¨a˜w˜w q™zœzÐ9Ð9Ð9Ð9Ð9Ð9r   zItem z is not a sequence: z Coefficient list cannot be empty)r   r*  r   rF   rë   rÿ   r  Úkeysr‰   r<   Ú	TypeErrorr>  r«   r   )
r·   r0  ÚvalidÚseqr:   r¯   rB  r@  r=  r?  s
          @@@r   r©   r©   û  s
  øøø€ ð
 SÐR€FØOÐO€Gð,•H�U”Oð ,­­eµU½EÐ.AÔ(Bð ,ð ,ð ,ð ,ð ,ð ,ð ,õ �%�ÑÔð 0Ø˜ÐÐØ—I’I�f W§\¢\¡^¤^Ñ4Ô4Ñ5Ô5ˆEÝÐ]¸UÐ]Ð]ÐV[Ð]Ð]Ñ^Ô^Ð^Ø�eŒnˆØˆv�c‰{Œ{ð 	F�c #™hœh¨!šm˜mÝÐD¨ÐDÐDÐDÑEÔEÐEØ/Ð/Ð/Ð/¨3Ð/Ñ/Ô/Ð/àˆ6�%‰=Œ=ð 
Ýð,ñ
ô 
ð 	
õ ˆ5�z„z�Q‚€�3Ð9Ð9Ð9Ð9°5Ð9Ñ9Ô9Ñ9Ô9€Ø�˜‘”Ð Ð ð €CÝ˜UÑ#Ô#ð #ð #‰ˆˆ4Øˆv�d‰|Œ|ð 	EÝÐC AÐCÐC¸4ÐCÐCÑDÔDÐDØ�
Š
�8�8˜D‘>”>Ñ"Ô"Ð"Ð"Øð =ÝÐ;Ñ<Ô<Ð<Ø€Jr   )F)rZ   )rm   rn   F)r„   )9r'  r  rj   r7  rÑ   Útypingr   r   r   r   r   r   rG   Útorch.distributed.tensorr	   rE   ÚImportErrorÚ_helpersr   r   r   r   r   Ú_typesr   r   r   Ú	getLoggerr$  r  r)  r(  rª   rë   r*  r-  ÚTensorr+  r   r,  r)   Úbfloat16r/   rY   ÚSizerf   rl   rƒ   r�   rž   r¥   r�   r¤   ÚoptimÚ	OptimizerrÆ   r©   r&   r   r   ú<module>rT     sº  ððð ð, €€€Ø €€€Ø €€€Ø €€€Ø BÐ BÐ BÐ BÐ BÐ BÐ BÐ BÐ BÐ BÐ BÐ BÐ BÐ BÐ BÐ Bà €€€ðØ0Ð0Ð0Ð0Ð0Ð0Ø€K€KøØð ð ð Ø€K€K€Kðøøøð kÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jØ Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à
ˆ'Ô
˜HÑ
%Ô
%€ð €ØÐ ð
 	"ððð ð ðð ð ðð ð ð5&ð &€ðR ��U˜5 %¨Ð.Ô/°°e¸EÀ5È%Ð<OÔ6PÔ1QÐQÔ
R€ðFð F�e”lð F°ð FÀÄð Fð Fð Fð Fð%Øð%à�S˜#�XŒð%ð ð%ð %ð %ð %ð8 Ø"Ø"œ^ØðVð VØŒ<ðVàðVð ˜5 ¨¨uÐ!4Ô5Ô6ðVð ð	Vð
 ðVð Œ{ðVð ðVð „\ðVð Vð Vð Vðv .ðLð LØ”ZðLàðLð ðLð Lð Lð Lð> .ðOð OØ”ZðOàðOð ˆ5�$ˆ;ÔðOð Oð Oð Oð< Ø"&Ø#ð	E3ð E3ØŒ|ðE3àðE3ð  ðE3ð ð	E3ð
 ˆ4��t˜S�yÔ!Ð!Ô"ðE3ð E3ð E3ð E3ðT ð 2ð  2Ø”ð 2àð 2ð ˆ5Œ<˜œÐ#Ô$ð 2ð  2ð  2ð  2ðh !ð#%ð %ð %Ø�U”\Ô"ð%à�E”LÔ!ð%ð ˜EœLÔ)ð%ð
 ð%ð ð%ð ð%ð ð%ð ð%ð !ð%ð ð%ð ð%ð ˜s”mð%ð ð%ð  ð%ð  ð!%ð" ð#%ð$ 
ð%%ð %ð %ð %ðx !ð)2ð 2ð 2Ø�U”\Ô"ð2à�E”LÔ!ð2ð ˜EœLÔ)ð2ð ˜%œ,Ô'ð	2ð
 ˜%œ,Ô'ð2ð ð2ð ð2ð ð2ð ð2ð ð2ð ð2ð !ð2ð ð2ð ð2ð  ˜s”mð!2ð" ð#2ð$  ð%2ð& ð'2ð( ð)2ð* 
ð+2ð 2ð 2ð 2ðL !ð#C7ð C7ð C7Ø�U”\Ô"ðC7à�E”LÔ!ðC7ð ˜EœLÔ)ðC7ð
 ðC7ð ðC7ð ðC7ð ðC7ð ðC7ð !ðC7ð ðC7ð ðC7ð ˜s”mðC7ð ðC7ð  ðC7ð  ð!C7ð" ð#C7ð$ 
ð%C7ð C7ð C7ð C7ðt !ð)t:ð t:ð t:Ø�U”\Ô"ðt:à�E”LÔ!ðt:ð ˜EœLÔ)ðt:ð ˜%œ,Ô'ð	t:ð
 ˜%œ,Ô'ðt:ð ðt:ð ðt:ð ðt:ð ðt:ð ðt:ð ðt:ð !ðt:ð ðt:ð ðt:ð  ˜s”mð!t:ð" ð#t:ð$  ð%t:ð& ð't:ð( ð)t:ð* 
ð+t:ð t:ð t:ð t:ðnjð jð jð jð jˆ5Œ;Ô ñ jô jð jðZ)Ø�S˜( 5œ/¨8°H¸U´OÔ+DÐDÔEð)à˜˜h x°¤Ô7Ð7Ô8ð)ð 
ˆ%��u˜eÐ#Ô
$Ô%ð)ð )ð )ð )ð )ð )s   ¨1 ±;º;