§
    l¬¸j¨/  ã                  ód   — d dl mZ d dlZd dlmZ ddd
„Z	 	 ddd„Z G d„ dej        ¦  «        ZdS )é    )ÚannotationsN)ÚoptimçH¯¼šò×z>ÚGútorch.TensorÚepsÚfloatÚreturnc                óð  — | j         dv sJ ‚|                      d|                      d¦  «        |                      d¦  «        ¦  «                             ¦   «         }||                     dd¬¦  «        |z   z  }|                      d¦  «        |                      d¦  «        k    r|                     dd¦  «        }d\  }}}t          d¦  «        D ]L}||                     dd¦  «        z  }t          j        |||||¬	¦  «        }t          j        ||||¬
¦  «        }ŒM|                      d¦  «        |                      d¦  «        k    r|                     dd¦  «        }|                     | j	        ¦  «        S )a<  Compute the zeroth power / orthogonalization of matrix G using Newton-Schulz iteration.

    This function implements a quintic Newton-Schulz iteration to compute an approximate orthogonalization of the input
    matrix G. The iteration coefficients are optimized to maximize convergence slope at zero, producing a result similar
    to UV^T from SVD, where USV^T = G, but with relaxed convergence guarantees that empirically work well for
    optimization purposes.

    Args:
        G (torch.Tensor): Input 2D matrix or 3D batch of matrices to orthogonalize.
        eps (float, optional): Small epsilon value added to norm for numerical stability. Default: 1e-7.

    Returns:
        (torch.Tensor): Orthogonalized matrix/matrices with same shape as input G.

    Examples:
        >>> G = torch.randn(128, 64)
        >>> G_ortho = zeropower_via_newtonschulz5(G)
        >>> print(G_ortho.shape)
        torch.Size([128, 64])

    Notes:
        - Uses bfloat16 precision for computation.
        - Performs exactly 5 Newton-Schulz iteration steps with fixed coefficients.
        - Automatically transposes for efficiency when rows > columns.
        - Output approximates US'V^T where S' has diagonal entries ~ Uniform(0.5, 1.5).
        - Does not produce exact UV^T but works well empirically for neural network optimization.
    >   é   é   éÿÿÿÿéþÿÿÿ)r   r   T)ÚdimÚkeepdim)gu“VŽ@gš™™™™Àg�—nƒ@ @é   )ÚbetaÚalpha)r   )
ÚndimÚreshapeÚsizeÚbfloat16ÚnormÚ	transposeÚrangeÚtorchÚbaddbmmÚshape)	r   r   ÚXÚaÚbÚcÚ_ÚAÚBs	            úf/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/ultralytics/optim/muon.pyÚzeropower_via_newtonschulz5r'   	   sO  € ð8 Œ6�VÐÐÐÐØ	�	Š	�"�a—f’f˜R‘j”j !§&¢&¨¡*¤*Ñ-Ô-×6Ò6Ñ8Ô8€AØˆ�Š�H dˆÑ	+Ô	+¨cÑ	1Ñ1€AØ‡v‚vˆb�z„z�A—F’F˜2‘J”JÒÐØ�KŠK˜˜BÑÔˆØ%�G€A€qˆ!Ý�1‰XŒXð +ð +ˆØ�—’˜B Ñ#Ô#Ñ#ˆÝŒM˜!˜Q ¨°Ð3Ñ3Ô3ˆÝŒM˜!˜Q ¨Ð*Ñ*Ô*ˆˆØ‡v‚vˆb�z„z�A—F’F˜2‘J”JÒÐØ�KŠK˜˜BÑÔˆØ�9Š9�Q”WÑÔÐó    çffffffî?TÚgradú!torch.Tensor | list[torch.Tensor]Úmomentumr   ÚnesterovÚboolc                ó°  ‡— t          | t          j        ¦  «        }|r| g|gfn| |f\  }}t          j        ||¦  «         t          j        ||d|z
  ¬¦  «         |r=t          t          j        ||¦  «        ¦  «        }t          j        ||d|z
  ¬¦  «         nt          |¦  «        }i }t          |¦  «        D �]V\  }	}
|
                     ¦   «         }| o*|
j	        dk    o|
                     t          j
        ¬¦  «        }|r+|
                     dddd¦  «                             d¦  «        n3|
j	        dk    r|
                     d¦  «        n|
                     ¦   «         }t          d||	                              d¦  «        ||	                              d	¦  «        z  ¦  «        d
z  }|                     |                     d¦  «        ||j        |j        fg ¦  «                             |	||s|r|
                     ¦   «         ndf¦  «         �ŒXg }|                     ¦   «         D ]¼\  }}|                     d„ ¬¦  «         d}t/          dt1          |¦  «        dz   ¦  «        D ]}}|t1          |¦  «        k    sG||         d                              d¦  «        d||         d                              d¦  «        z  k    r!|                     ||||…         f¦  «         |}Œ~Œ½|D �]d\  \  }}}}}t          j        t1          |¦  «        |d         d                              d¦  «        |||¬¦  «        Št          j        ˆfd„t          |¦  «        D ¦   «         d„ |D ¦   «         ¦  «         t5          ‰¦  «                             ¦   «                              ||d         d                  j        ¦  «                             |¦  «        Št          |¦  «        D ]n\  }\  }	}}‰|d|                     d¦  «        …f         }|r!|                     ||	         j        |¦  «        n|                     ||	         j        ¦  «        ||	<   Œo�Œf|r|d         n|S )aò  Compute Muon optimizer updates with momentum and orthogonalization.

    This function applies momentum to the gradients, optionally uses Nesterov acceleration, and then orthogonalizes the
    updates using Newton-Schulz iterations. Matrices with the same column count are zero-padded and orthogonalized in a
    single batched call, and momentum math uses fused foreach ops, avoiding per-parameter kernel launch overhead.
    Higher-rank tensors are reshaped before orthogonalization, and each update is scaled based on parameter dimensions.

    Args:
        grad (torch.Tensor | list[torch.Tensor]): Gradient tensor(s) to update. Each must have at least two dimensions.
        momentum (torch.Tensor | list[torch.Tensor]): Momentum buffer tensor(s), modified in-place.
        beta (float, optional): Momentum coefficient for exponential moving average. Default: 0.95.
        nesterov (bool, optional): Whether to use Nesterov momentum acceleration. Default: True.

    Returns:
        (torch.Tensor | list[torch.Tensor]): Orthogonalized update tensor(s), each with the gradient's shape and dtype.

    Examples:
        >>> grad = torch.randn(64, 128)
        >>> momentum = torch.zeros_like(grad)
        >>> update = muon_update(grad, momentum, beta=0.95, nesterov=True)
        >>> print(update.shape)
        torch.Size([64, 128])

    Notes:
        - Momentum buffers are updated in-place: momentum = beta * momentum + (1-beta) * grad.
        - With Nesterov: update = beta * momentum + (1-beta) * grad.
        - Without Nesterov: update = momentum.
        - Tensors with more than 2 dimensions are reshaped to 2D with the first dimension preserved.
        - Final updates are scaled by sqrt(max(1, dim[-2] / dim[-1])) to account for parameter dimensions.
    é   ©r   é   )Úmemory_formatr   r   r   r   r   ç      à?Nc                ó:   — | d                               d¦  «         S )Nr0   r   ©r   )Úts    r&   ú<lambda>zmuon_update.<locals>.<lambda>o   s   €  ! A¤$§)¢)¨A¡,¤, € r(   )Úkeyé   )ÚdeviceÚdtypec                ó\   •— g | ](\  }\  }}}‰|d |                      d¦  «        …f         ‘Œ)S )Nr   r6   )Ú.0Újr#   Úmr   s       €r&   ú
<listcomp>zmuon_update.<locals>.<listcomp>y   s:   ø€ ÐTÐTÐT±<°1±i°q¸!¸Q˜Q˜q + A§F¢F¨1¡I¤I +˜~Ô.ÐTÐTÐTr(   c                ó   — g | ]\  }}}|‘Œ	S © rC   )r>   r#   r@   s      r&   rA   zmuon_update.<locals>.<listcomp>y   s#   € ÐVnÐVnÐVnÑ]dÐ]^Ð`aÐcdÐWXÐVnÐVnÐVnr(   ) Ú
isinstancer   ÚTensorÚ_foreach_mul_Ú_foreach_add_ÚlistÚ_foreach_mulÚ	enumerateÚis_contiguousr   Úchannels_lastÚpermuteÚflattenÚ
contiguousÚmaxr   Ú
setdefaultr;   r<   ÚappendÚstrideÚitemsÚsortr   ÚlenÚzerosr'   ÚtoÚmul_Ú
as_stridedr   r   )r*   r,   r   r-   ÚsingleÚgradsÚ	momentumsÚupdatesÚbucketsÚiÚuÚdenseÚnhwcr@   ÚscaleÚgroupsr9   rT   Ústartr?   Úcolsr;   r<   rS   Úxr   s                            @r&   Úmuon_updateri   4   sP  ø€ õH ˜�eœlÑ+Ô+€FØ/5ÐK˜˜  
Ð+Ð+¸DÀ(Ð;KÑ€Eˆ9Ý	Ô˜	 4Ñ(Ô(Ð(Ý	Ô˜	 5°°D±Ð9Ñ9Ô9Ð9Øð "Ý•uÔ)¨)°TÑ:Ô:Ñ;Ô;ˆÝÔ˜G U°!°d±(Ð;Ñ;Ô;Ð;Ð;å�y‘/”/ˆØ€GÝ˜'Ñ"Ô"ð 

ñ 

‰ˆˆ1ð —’Ñ!Ô!ˆØˆyÐ_˜QœV qš[Ð_¨Q¯_ª_Í5ÔK^¨_Ñ-_Ô-_ˆØ04ÐjˆA�IŠI�a˜˜A˜qÑ!Ô!×)Ò)¨!Ñ,Ô,Ð,È1Ì6ÐTUÊ:È:¸1¿9º9ÀQ¹<¼<¸<Ð[\×[gÒ[gÑ[iÔ[iˆÝ�A�u˜Q”x—}’} RÑ(Ô(¨5°¬8¯=ª=¸Ñ+<Ô+<Ñ<Ñ=Ô=ÀÑDˆØ×Ò˜AŸFšF 1™IœI u¨a¬h¸¼Ð@À"ÑEÔE×LÒLØ� Ð8¨$Ð8�1—8’8‘:”:�:°DÐ9ñ	
ô 	
ð 	
ñ 	
ð €FØ—m’m‘o”oð ð ‰
ˆˆUØ�
Š
Ð.Ð.ˆ
Ñ/Ô/Ð/ØˆÝ�q�#˜e™*œ* q™.Ñ)Ô)ð 	ð 	ˆAØ•C˜‘J”JŠˆ %¨¤,¨q¤/×"6Ò"6°qÑ"9Ô"9¸BÀÀqÄÈ!Ä×AQÒAQÐRSÑATÔATÑ<TÒ"TÐ"TØ—’˜s E¨%°¨'¤NÐ3Ñ4Ô4Ð4Ø�øð	ð 06ð gñ gÑ+Ñ$ˆˆu�f˜e eõ ŒK�˜E™
œ
 E¨!¤H¨Q¤K×$4Ò$4°QÑ$7Ô$7¸ÀfÐTYÐZÑZÔZˆÝÔÐTÐTÐTÐTÅ9ÈUÑCSÔCSÐTÑTÔTÐVnÐVnÐhmÐVnÑVnÔVnÑoÔoÐoÝ'¨Ñ*Ô*×5Ò5Ñ7Ô7×:Ò:¸5ÀÀqÄÈ!ÄÔ;MÔ;SÑTÔT×YÒYÐZ_Ñ`Ô`ˆÝ!*¨5Ñ!1Ô!1ð 	gð 	gÑˆA‰~��1�fØ�!�[�q—v’v˜a‘y”y�[�.Ô!ˆAØAGÐf˜Ÿš e¨A¤h¤n°fÑ=Ô=Ð=ÈQÏYÊYÐW\Ð]^ÔW_ÔWeÑMfÔMfˆG�A‰JˆJñ	gð  Ð,ˆ7�1Œ:ˆ: WÐ,r(   c                  óf   ‡ — e Zd ZdZ	 	 	 	 	 	 	 ddˆ fd„Z ej        ¦   «         dd„¦   «         Zˆ xZS )ÚMuSGDa"  Hybrid optimizer combining Muon and SGD updates for neural network training.

    This optimizer implements a combination of Muon (a momentum-based optimizer with orthogonalization via Newton-Schulz
    iterations) and standard SGD with momentum. It allows different parameter groups to use either the hybrid Muon+SGD
    approach or pure SGD.

    Args:
        params (Iterable): Parameters to optimize or dicts defining parameter groups.
        muon (float, optional): Weight factor for Muon updates in hybrid mode. Default: 0.5.
        sgd (float, optional): Weight factor for SGD updates in hybrid mode. Default: 0.5.

    Attributes:
        muon (float): Scaling factor applied to Muon learning rate.
        sgd (float): Scaling factor applied to SGD learning rate in hybrid mode.

    Examples:
        >>> param_groups = [
        ...     {
        ...         "params": model.conv_params,
        ...         "lr": 0.02,
        ...         "use_muon": True,
        ...         "momentum": 0.95,
        ...         "nesterov": True,
        ...         "weight_decay": 0.01,
        ...     },
        ...     {
        ...         "params": model.other_params,
        ...         "lr": 0.01,
        ...         "use_muon": False,
        ...         "momentum": 0.9,
        ...         "nesterov": False,
        ...         "weight_decay": 0,
        ...     },
        ... ]
        >>> optimizer = MuSGD(param_groups, muon=0.5, sgd=0.5)
        >>> loss = model(data)
        >>> loss.backward()
        >>> optimizer.step()

    Notes:
        - Parameter groups with 'use_muon': True will receive both Muon and SGD updates.
        - Parameter groups with 'use_muon': False will receive only SGD updates.
        - The Muon update uses orthogonalization which works best for 2D+ parameter tensors.
    çü©ñÒMbP?ç        Fr4   Úlrr	   r,   Úweight_decayr-   r.   Úuse_muonÚmuonÚsgdc	                óx   •— |||||dœ}	t          ¦   «                              ||	¦  «         || _        || _        dS )aN  Initialize MuSGD optimizer with hybrid Muon and SGD capabilities.

        Args:
            params (Iterable): Iterable of parameters to optimize or dicts defining parameter groups.
            lr (float): Learning rate.
            momentum (float): Momentum factor for SGD.
            weight_decay (float): Weight decay (L2 penalty).
            nesterov (bool): Whether to use Nesterov momentum.
            use_muon (bool): Whether to enable Muon updates.
            muon (float): Scaling factor for Muon component.
            sgd (float): Scaling factor for SGD component.
        )rn   r,   ro   r-   rp   N)ÚsuperÚ__init__rq   rr   )ÚselfÚparamsrn   r,   ro   r-   rp   rq   rr   ÚdefaultsÚ	__class__s             €r&   ru   zMuSGD.__init__¯   sN   ø€ ð0 Ø Ø(Ø Ø ð
ð 
ˆõ 	‰Œ×Ò˜ Ñ*Ô*Ð*ØˆŒ	ØˆŒˆˆr(   Nc                óÞ  ‡ — d}|�5t          j        ¦   «         5   |¦   «         }ddd¦  «         n# 1 swxY w Y   ‰ j        D �]©}d„ |d         D ¦   «         }|sŒ|d         |d         |d         }}}|D ]l}t          ‰ j        |         ¦  «        dk    rLt          j        |¦  «        ‰ j        |         d<   |d	         r"t          j        |¦  «        ‰ j        |         d
<   Œm|d	         rbt          d„ |D ¦   «         ˆ fd„|D ¦   «         ||¬¦  «        }	t          j        ||	|‰ j        z   ¬¦  «         ˆ fd„|D ¦   «         }
|‰ j	        z  }nˆ fd„|D ¦   «         }
d„ |D ¦   «         }|d         dk    rt          j
        |||d         ¬¦  «        }t          j        |
|¦  «         t          j        |
|¦  «         |rt          j
        ||
|¬¦  «        n|
}	t          j        ||	| ¬¦  «         �Œ«|S )a#  Perform a single optimization step.

        Applies either hybrid Muon+SGD updates or pure SGD updates depending on the
        'use_muon' flag in each parameter group. For Muon-enabled groups, parameters
        receive both an orthogonalized Muon update and a standard SGD momentum update.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss. Default: None.

        Returns:
            (torch.Tensor | None): The loss value if closure is provided, otherwise None.

        Notes:
            - Parameters with None gradients are skipped.
            - Muon updates use Newton-Schulz orthogonalization and work best on 2D+ tensors.
            - Weight decay is applied only to the SGD component in hybrid mode.
        Nc                ó    — g | ]}|j         ®	|‘ŒS ©N©r*   ©r>   Úps     r&   rA   zMuSGD.step.<locals>.<listcomp>ë   s   € ÐGÐGÐG˜A°A´FÐ4F�aÐ4FÐ4FÐ4Fr(   rw   rn   r,   r-   r   Úmomentum_bufferrp   Úmomentum_buffer_SGDc                ó   — g | ]	}|j         ‘Œ
S rC   r}   r~   s     r&   rA   zMuSGD.step.<locals>.<listcomp>ö   ó   € Ð,Ð,Ð, �Q”VÐ,Ð,Ð,r(   c                ó6   •— g | ]}‰j         |         d          ‘ŒS ©r€   ©Ústate©r>   r   rv   s     €r&   rA   zMuSGD.step.<locals>.<listcomp>÷   s%   ø€ ÐFÐFÐF¸!�T”Z ”]Ð#4Ô5ÐFÐFÐFr(   )r   r-   r1   c                ó6   •— g | ]}‰j         |         d          ‘ŒS )r�   r†   rˆ   s     €r&   rA   zMuSGD.step.<locals>.<listcomp>ü   s%   ø€ ÐPÐPÐPÀA˜4œ: aœ=Ð)>Ô?ÐPÐPÐPr(   c                ó6   •— g | ]}‰j         |         d          ‘ŒS r…   r†   rˆ   s     €r&   rA   zMuSGD.step.<locals>.<listcomp>ÿ   s%   ø€ ÐLÐLÐLÀ˜4œ: aœ=Ð):Ô;ÐLÐLÐLr(   c                ó   — g | ]	}|j         ‘Œ
S rC   r}   r~   s     r&   rA   zMuSGD.step.<locals>.<listcomp>  rƒ   r(   ro   )r   Úenable_gradÚparam_groupsrV   r‡   Ú
zeros_likeri   rG   rq   rr   Ú_foreach_addrF   )rv   ÚclosureÚlossÚgrouprw   rn   r,   r-   r   r^   Úbuffersr\   s   `           r&   Ústepz
MuSGD.stepÑ   s�  ø€ ð( ˆØÐÝÔ"Ñ$Ô$ð !ð !Ø�w‘y”y�ð!ð !ð !ñ !ô !ð !ð !ð !ð !ð !ð !øøøð !ð !ð !ð !ð Ô&ð 	<ñ 	<ˆEØGÐG  x¤ÐGÑGÔGˆFØð ØØ%*¨4¤[°%¸
Ô2CÀUÈ:ÔEV˜(�ˆBØð Sð S�Ý�t”z !”}Ñ%Ô%¨Ò*Ð*Ý7<Ô7GÈÑ7JÔ7J�D”J˜q”MÐ"3Ñ4Ø˜ZÔ(ð SÝ?DÔ?OÐPQÑ?RÔ?R˜œ
 1œÐ&;Ñ<øØ�ZÔ ð MÝ%Ø,Ð, VÐ,Ñ,Ô,ØFÐFÐFÐF¸vÐFÑFÔFØ!Ø%ð	ñ ô �õ Ô# F¨G¸RÀ$Ä)¹^Ð;LÐMÑMÔMÐMØPÐPÐPÐPÈÐPÑPÔP�Ø�d”h‘��àLÐLÐLÐLÀVÐLÑLÔL�à,Ð, VÐ,Ñ,Ô,ˆEØ�^Ô$¨Ò)Ð)ÝÔ*¨5°&ÀÀnÔ@UÐVÑVÔV�ÝÔ ¨Ñ2Ô2Ð2ÝÔ ¨Ñ/Ô/Ð/ØLTÐa•eÔ(¨°¸xÐHÑHÔHÐHÐZaˆGÝÔ ¨¸°sÐ;Ñ;Ô;Ð;Ñ;Øˆs   ™0°4·4)rl   rm   rm   FFr4   r4   )rn   r	   r,   r	   ro   r	   r-   r.   rp   r.   rq   r	   rr   r	   r|   )	Ú__name__Ú
__module__Ú__qualname__Ú__doc__ru   r   Úno_gradr”   Ú__classcell__)ry   s   @r&   rk   rk   �   s�   ø€ € € € € ð+ð +ð` ØØ!ØØØØð ð  ð  ð  ð  ð  ð  ðD €U„]�_„_ð6ð 6ð 6ñ „_ð6ð 6ð 6ð 6ð 6r(   rk   )r   )r   r   r   r	   r
   r   )r)   T)
r*   r+   r,   r+   r   r	   r-   r.   r
   r+   )Ú
__future__r   r   r   r'   ri   Ú	Optimizerrk   rC   r(   r&   ú<module>r�      sµ   ðð #Ð "Ð "Ð "Ð "Ð "à €€€Ø Ð Ð Ð Ð Ð ð(ð (ð (ð (ð (ð\ Øð	J-ð J-ð J-ð J-ð J-ðZGð Gð Gð Gð GˆEŒOñ Gô Gð Gð Gð Gr(   