§
    9²¸jH”  ã                  óp  — U d dl mZ d dlZd dlZddlmZ ddlmZ ddlm	Z	m
Z
  ej        e¦  «        Z e	¦   «         rd dlZ e
¦   «         rd dlmZ d dlmZmZmZmZmZ d dlmZ dFd„ZdGd„ZdHdId„Zej        d„ ¦   «         Z G d„ d¦  «        Z G d„ de¦  «        Z  G d„ de¦  «        Z! G d„ de¦  «        Z" G d „ d!e¦  «        Z# G d"„ d#e¦  «        Z$ G d$„ d%e¦  «        Z% G d&„ d'e¦  «        Z& G d(„ d)e¦  «        Z' G d*„ d+e¦  «        Z( e
¦   «         r0 G d,„ d-ej)        j*        ¦  «        Z+ G d.„ d/ej)        j*        ¦  «        Z, G d0„ d1e¦  «        Z- G d2„ d3e¦  «        Z. G d4„ d5e¦  «        Z/ G d6„ d7e/¦  «        Z0 G d8„ d9e-¦  «        Z1 G d:„ d;e¦  «        Z2 e2¦   «         Z3d;e4d<<   dJd>„Z5d?„ Z6dKdE„Z7dS )Lé    )ÚannotationsNé   )Úlogging)ÚGeneralInterface)Úis_torch_availableÚis_torch_distributed_available)ÚDTensorÚPartialÚ	ReplicateÚShardÚdistribute_tensor)Ú_StridedShardÚnameÚstrÚreturnc                ó0   — t          j        dd„ | ¦  «        S )ag  
    Replace the numbers in the `name` by wildcards, only if they are in-between dots (`.`) or if they are between
    a dot (`.`) and the end of the string.
    This matches how modules are named/numbered when using a nn.ModuleList or nn.Sequential, but will NOT match
    numbers in a parameter name itself, e.g. if the param is named `"w1"` or `"w2"`.
    z\.\d+(\.|$)c                ó2   — d|                       d¦  «        z   S )Nz.*é   ©Úgroup)Úms    úx/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/distributed/tensor_parallel.pyú<lambda>z2replace_layer_number_by_wildcard.<locals>.<lambda>*   s   € ¨D°1·7²7¸1±:´:Ñ,=€ ó    )ÚreÚsub)r   s    r   Ú replace_layer_number_by_wildcardr   #   s   € õ Œ6�.Ð"=Ð"=¸tÑDÔDÐDr   Úexpected_keysú	list[str]Útp_planúdict[str, str] | Nonec                óÎ  — |€dS d„ | D ¦   «         }t          |¦  «        }|                     ¦   «         }|D ]¹}d|v r|                     dd¦  «        d         n|}t          j        dd|¦  «        }||v r,|                     |d¦  «         |                     |¦  «         Œjd|v rK|                     dd¦  «        d         x}|v r+|                     |d¦  «         |                     |¦  «         Œºt          |¦  «        dk    rt           	                    d|› �¦  «         t          |¦  «        dk    r2t           	                    d	d
 
                    |¦  «        › �¦  «         dS dS )z�
    Verify the TP plan of the model, log a warning if the layers that were not sharded and the rules that were not applied.
    Nc                ó,   — h | ]}t          |¦  «        ’ŒS © )r   )Ú.0Úkeys     r   ú	<setcomp>z!verify_tp_plan.<locals>.<setcomp>5   s!   € ÐSÐSÐS¸cÕ4°SÑ9Ô9ÐSÐSÐSr   ú.r   r   z\d+Ú*z>The following TP rules were not applied on any of the layers: z'The following layers were not sharded: z, )ÚsetÚcopyÚrsplitr   r   ÚpopÚdiscardÚlenÚloggerÚwarningÚjoin)	r   r    Úgeneric_keysÚunsharded_layersÚunused_rulesr&   Ú
param_nameÚgeneric_param_nameÚparent_param_names	            r   Úverify_tp_planr9   -   s˜  € ð
 €ØˆàSÐSÀ]ÐSÑSÔS€LÝ˜<Ñ(Ô(ÐØ—<’<‘>”>€Làð 	*ð 	*ˆØ.1°S¨j¨j�S—Z’Z  QÑ'Ô'¨Ô*Ð*¸cˆ
ÝœV F¨C°Ñ<Ô<Ðà Ð(Ð(Ø×ÒÐ/°Ñ6Ô6Ð6Ø×$Ò$ SÑ)Ô)Ð)Ð)ØÐ&Ð&Ð&ÐAS×AZÒAZÐ[^Ð`aÑAbÔAbÐcdÔAeÐ,eÐ,=ÐjqÐ+qÐ+qØ×ÒÐ.°Ñ5Ô5Ð5Ø×$Ò$ SÑ)Ô)Ð)øå
ˆ<ÑÔ˜1ÒÐÝ�ŠÐfÐXdÐfÐfÑgÔgÐgÝ
ÐÑÔ˜qÒ Ð Ý�ŠÐ^ÀÇÂÐK[ÑA\ÔA\Ð^Ð^Ñ_Ô_Ð_Ð_Ð_ð !Ð r   TÚparameter_nameúdict[str, str]ú
str | Nonec                ó˜   — t          | ¦  «        }||v r||         S |r,d|v r(|                     dd¦  «        d         x}|v r||         S dS )a£  
    Get the TP style for a parameter from the TP plan.

    The TP plan is a dictionary that maps parameter names to TP styles.
    The parameter name can be a generic name with wildcards (e.g. "*.weight") or a specific name (e.g. "layer_1.weight").

    The `is_weight` is important because for weights, we want to support `.weights` and `.bias` cases seamlessly! but
    not parent classes for `post_init` calls
    r(   r   r   N)r   r,   )r:   r    Ú	is_weightr7   Úmodule_names        r   Ú_get_parameter_tp_planr@   J   su   € õ :¸.ÑIÔIÐØ˜WÐ$Ð$ØÐ)Ô*Ð*Ø	ð $�sÐ0Ð0Ð0ÐEW×E^ÒE^Ð_bÐdeÑEfÔEfÐghÔEiÐ6i°kÐnuÐ5uÐ5uØ�{Ô#Ð#Øˆ4r   c           	   #  óZ  K  — d„ |                       d¬¦  «        D ¦   «         }d„ |                     ¦   «         D ¦   «         }| j                             |¦  «         	 d V — |                     ¦   «         D ]Ž\  }}| j        |         }|||         u r|| j        |<   Œ'|�et	          |t
          ¦  «        sPt          j        ||j        |j        d¬¦  «        }t          j
                             ||j        ¬¦  «        | j        |<   Œ�d S # |                     ¦   «         D ]Ž\  }}| j        |         }|||         u r|| j        |<   Œ'|�et	          |t
          ¦  «        sPt          j        ||j        |j        d¬¦  «        }t          j
                             ||j        ¬¦  «        | j        |<   Œ�w xY w)Nc                óD   — i | ]\  }}t          |t          ¦  «        ¯||“ŒS r$   )Ú
isinstancer	   ©r%   r   Úparams      r   ú
<dictcomp>z-_use_local_dtensor_params.<locals>.<dictcomp>`   s0   € ÐuÐuÐu¡  uÕZdÐejÕlsÑZtÔZtÐu��uÐuÐuÐur   F©Úrecursec                ó>   — i | ]\  }}||                      ¦   «         “ŒS r$   )Úto_localrD   s      r   rF   z-_use_local_dtensor_params.<locals>.<dictcomp>a   s(   € ÐPÐPÐP©{¨t°U�D˜%Ÿ.š.Ñ*Ô*ÐPÐPÐPr   ©Ú	run_check©Úrequires_grad)Únamed_parametersÚitemsÚ_parametersÚupdaterC   r	   Ú
from_localÚdevice_meshÚ
placementsÚtorchÚnnÚ	ParameterrN   )ÚmoduleÚ	originalsÚlocal_paramsr   ÚoriginalÚcurrentÚreplacements          r   Ú_use_local_dtensor_paramsr_   \   sö  è è € ð vÐu°×0GÒ0GÐPUÐ0GÑ0VÔ0VÐuÑuÔu€IØPÐP¸i¿oºoÑ>OÔ>OÐPÑPÔP€LØ
Ô×Ò˜lÑ+Ô+Ð+ðØˆˆˆà'ŸošoÑ/Ô/ð 	ð 	‰NˆD�(ØÔ(¨Ô.ˆGà˜, tÔ,Ð,Ð,Ø+3�Ô" 4Ñ(Ð(ð Ð$­Z¸ÅÑ-IÔ-IÐ$Ý%Ô0ØØÔ(ØÔ'Ø#ð	ñ ô �õ ,1¬8×+=Ò+=ØØ")Ô"7ð ,>ñ ,ô ,�Ô" 4Ñ(øð	ð 	ø˜iŸošoÑ/Ô/ð 	ð 	‰NˆD�(ØÔ(¨Ô.ˆGà˜, tÔ,Ð,Ð,Ø+3�Ô" 4Ñ(Ð(ð Ð$­Z¸ÅÑ-IÔ-IÐ$Ý%Ô0ØØÔ(ØÔ'Ø#ð	ñ ô �õ ,1¬8×+=Ò+=ØØ")Ô"7ð ,>ñ ,ô ,�Ô" 4Ñ(øð	øøøs   ÁD ÄB%F*c                  ó:   — e Zd Zd„ Zd	d„Zd„ Zd„ Zd„ Zd„ Zd„ Z	dS )
ÚTensorParallelLayerc                ó   — dS )zCWhether this module's forward requires local inputs and parameters.Fr$   ©ÚselfrY   s     r   Úshould_use_local_tensorsz,TensorParallelLayer.should_use_local_tensors|   s   € àˆur   Nc                ó   — dS )z3Validate a parameter before applying this TP style.Nr$   )rd   rY   rE   Úmeshr:   s        r   Úvalidate_paramz"TensorParallelLayer.validate_param€   ó   € àˆr   c                ó   — dS )z<Wrap ONE parameter as a DTensor placeholder. Default: no-op.Nr$   )rd   rY   rE   rg   s       r   Úshard_paramzTensorParallelLayer.shard_param„   ri   r   c                ó
   — ||fS ©Nr$   )rd   rY   ÚargsÚkwargsrg   s        r   Útransform_inputs_pre_forwardz0TensorParallelLayer.transform_inputs_pre_forwardˆ   s   € Ø�Vˆ|Ðr   c                óp   — |                       |¦  «        rt          |¦  «        S t          j        ¦   «         S rm   )re   r_   Ú
contextlibÚnullcontext©rd   rY   rg   s      r   Úcontext_around_forwardz*TensorParallelLayer.context_around_forward‹   s5   € Ø×(Ò(¨Ñ0Ô0ð 	5Ý,¨VÑ4Ô4Ð4ÝÔ%Ñ'Ô'Ð'r   c                ó   — |S rm   r$   ©rd   rY   Úoutputrg   s       r   Útransform_output_post_forwardz1TensorParallelLayer.transform_output_post_forward�   ó   € Øˆr   c                ó:   ‡ ‡‡‡— ‰j         Šˆˆˆˆ fd„}|‰_         ‰S )zCInstall pre / around / post transforms by replacing module.forward.c                 óÔ   •— ‰                      ‰| |‰¦  «        \  } }‰                     ‰‰¦  «        5   ‰| i |¤Ž}d d d ¦  «         n# 1 swxY w Y   ‰                     ‰|‰¦  «        S rm   ©rp   ru   ry   )rn   ro   rx   rg   rY   Úoriginal_forwardrd   s      €€€€r   Ú
tp_forwardz7TensorParallelLayer.install_forward.<locals>.tp_forward—   s¹   ø€ Ø×<Ò<¸VÀTÈ6ÐSWÑXÔX‰LˆD�&Ø×,Ò,¨V°TÑ:Ô:ð ;ð ;Ø)Ð)¨4Ð:°6Ð:Ð:�ð;ð ;ð ;ñ ;ô ;ð ;ð ;ð ;ð ;ð ;ð ;øøøð ;ð ;ð ;ð ;à×5Ò5°f¸fÀdÑKÔKÐKs   ³	AÁAÁA)Úforward)rd   rY   rg   r   r~   s   ``` @r   Úinstall_forwardz#TensorParallelLayer.install_forward“   sL   øøøø€ à!œ>Ðð	Lð 	Lð 	Lð 	Lð 	Lð 	Lð 	Lð 	Lð $ˆŒØˆr   rm   )
Ú__name__Ú
__module__Ú__qualname__re   rh   rk   rp   ru   ry   r�   r$   r   r   ra   ra   {   s‚   € € € € € ðð ð ðð ð ð ðð ð ðð ð ð(ð (ð (ð
ð ð ðð ð ð ð r   ra   c                  óD   — e Zd ZdZddddœdd„Zd„ Zdd	„Zd
„ Zd„ Zd„ Z	dS )ÚColwiseParallelub   Column-wise: weight & bias â†’ Shard(0) (Embedding: Shard(1)); input replicated, output Shard(-1).NT©Úinput_layoutsÚoutput_layoutsÚuse_local_outputrŠ   Úboolc               ón   — |pt          ¦   «         | _        |�|nt          d¦  «        | _        || _        d S ©Néÿÿÿÿ)r   rˆ   r   r‰   rŠ   ©rd   rˆ   r‰   rŠ   s       r   Ú__init__zColwiseParallel.__init__¤   s<   € Ø*Ð9­i©k¬kˆÔØ0>Ð0J˜n˜nÕPUÐVXÑPYÔPYˆÔØ 0ˆÔÐÐr   c                ó’   — t          |dd¦  «        }t          |t          j        j        ¦  «        ot          j        ¦   «          }|p|S ©NÚ_hf_quantized_needs_local_tpF©ÚgetattrrC   rV   rW   ÚLinearÚis_grad_enabled©rd   rY   Úuse_local_quantized_pathÚuses_local_inference_kernels       r   re   z(ColwiseParallel.should_use_local_tensors©   óH   € Ý#*¨6Ð3QÐSXÑ#YÔ#YÐ Ý&0°½¼¼Ñ&IÔ&IÐ&iÕRWÔRgÑRiÔRiÐNiÐ#Ø'ÐFÐ+FÐFr   c           	     ó–  — |j                              |¦  «        }t          | j        t          ¦  «        }|�|sd S t          |t
          j        j        ¦  «        rdn	|j        dz
  }|j	        |         }| 
                    ¦   «         }	||	z  dk    r9|p|}|                     dd¦  «        d         }
t          d|
› d|› d|	› d�¦  «        ‚d S )	Nr   r   r   r(   zThe output size of `z` (z1) must be divisible by the tensor parallel size (z") when gathering a colwise output.)rQ   ÚgetrC   r‰   r   rV   rW   Ú	EmbeddingÚndimÚshapeÚsizer,   Ú
ValueError)rd   rY   rE   rg   r:   ÚmetaÚgathers_outputÚ	shard_dimÚoutput_sizeÚtp_sizeÚ
layer_names              r   rh   zColwiseParallel.validate_param®   sú   € ØÔ!×%Ò% eÑ,Ô,ˆÝ# DÔ$7½ÑCÔCˆØˆ<˜~ˆ<ØˆFå# F­E¬HÔ,>Ñ?Ô?ÐR�A�AÀTÄYÐQRÁ]ˆ	Ø”j Ô+ˆØ—)’)‘+”+ˆØ˜Ñ  AÒ%Ð%Ø+Ð4¨uˆNØ'×.Ò.¨s°AÑ6Ô6°qÔ9ˆJÝð@ zð @ð @°kð @ð @Øð@ð @ð @ñô ð ð &Ð%r   c                óL  — |j                              |¦  «        }|€d S t          |t          j        j        ¦  «        rt          d¦  «        nt          |j        dz
  ¦  «        }t          j                             t          |||gd ¬¦  «        |j
        ¬¦  «        |j         |<   d S )Nr   r   ©Úsrc_data_rankrM   )rQ   r�   rC   rV   rW   rž   r   rŸ   rX   r   rN   ©rd   rY   rE   rg   r£   Ú	placements         r   rk   zColwiseParallel.shard_param¿   s�   € ØÔ!×%Ò% eÑ,Ô,ˆØˆ<ØˆFÝ *¨6µ5´8Ô3EÑ FÔ FÐ`•E˜!‘H”H�HÍEÐRVÔR[Ð^_ÑR_ÑL`ÔL`ˆ	Ý$)¤H×$6Ò$6Ý˜d D¨9¨+ÀTÐJÑJÔJØÔ,ð %7ñ %
ô %
ˆÔ˜5Ñ!Ð!Ð!r   c                ó  — |d         }t          |t          ¦  «         }|r6t          |t          j        j        ¦  «        rt          j        ¦   «         s||fS |rŠt          |dd¦  «        ryt          j        ¦   «         rV|j        rO|j        dk    r| 	                    ¦   «         n| 	                    d¦  «        }t                               ||¦  «        }|f|dd …         z   |fS |rt          j        ||| j        gd¬¦  «        }|j        t          ¦   «         fk    r#|                     t          ¦   «         g¬¦  «        }|                      |¦  «        r#|                     t'          ¦   «         g¬¦  «        }|f|dd …         z   |fS )	Nr   r“   Fr   ÚtprK   ©rU   ©Úgrad_placements)rC   r	   rV   rW   r–   r—   r•   rN   rŸ   Ú	get_groupÚ_AllReduceBackwardÚapplyrS   rˆ   rU   r   Úredistributere   rJ   r
   )rd   rY   rn   ro   rg   ÚxÚis_local_inputÚprocess_groups           r   rp   z,ColwiseParallel.transform_inputs_pre_forwardÉ   s…  € Ø�ŒGˆÝ'¨­7Ñ3Ô3Ð3ˆð ð 	 �j¨µ´´ÑAÔAð 	 Í%ÔJ_ÑJaÔJað 	 Ø˜�<Ðð
 ð 	+�g fÐ.LÈeÑTÔTð 	+ÝÔ$Ñ&Ô&ð ?¨1¬?ð ?Ø48´IÀ²N°N §¢Ñ 0Ô 0Ð 0ÈÏÊÐW[ÑH\ÔH\�Ý&×,Ò,¨Q°Ñ>Ô>�Ø�4˜$˜q˜r˜rœ(‘? FÐ*Ð*ð ð 	SÝÔ" 1 d¨TÔ-?Ð,@ÈEÐRÑRÔRˆAØŒ<�I™KœK˜>Ò)Ð)Ø—’­9©;¬;¨-�Ñ8Ô8ˆAØ×(Ò(¨Ñ0Ô0ð 	8Ø—
’
­G©I¬I¨;�
Ñ7Ô7ˆAØˆt�d˜1˜2˜2”h‰ Ð&Ð&r   c                óö  — t          |t          ¦  «         o>t          | j        t          ¦  «        o$| j        j        d|                     ¦   «         dz
  fv }|                      |¦  «        r| j        r|r|S t          |t          ¦  «        s&t          j        ||t          d¦  «        gd¬¦  «        }|j        | j        fk    r| 	                    | j        g¬¦  «        }| j        r| 
                    ¦   «         n|S )NrŽ   r   FrK   r°   )rC   r	   r‰   r   Údimre   rŠ   rS   rU   r¶   rJ   )rd   rY   rx   rg   Úoutput_is_local_shards        r   ry   z-ColwiseParallel.transform_output_post_forwardä   s  € õ ˜6¥7Ñ+Ô+Ð+ð BÝ˜4Ô.µÑ6Ô6ðBàÔ#Ô'¨B°·
²
±´¸qÑ0@Ð+AÐAð 	ð
 ×(Ò(¨Ñ0Ô0ð 	°TÔ5Jð 	ÐOdð 	ØˆMÝ˜&¥'Ñ*Ô*ð 	TÝÔ'¨°µu¸R±y´y°kÈUÐSÑSÔSˆFØÔ Ô!4Ð 6Ò6Ð6Ø×(Ò(°TÔ5HÐ4IÐ(ÑJÔJˆFØ$(Ô$9ÐEˆv�ŠÑ Ô Ð ¸vÐEr   ©rŠ   r‹   rm   )
r‚   rƒ   r„   Ú__doc__r�   re   rh   rk   rp   ry   r$   r   r   r†   r†   ¡   s˜   € € € € € ØlÐlà(,¸TÐ\`ð 1ð 1ð 1ð 1ð 1ð 1ð
Gð Gð Gð
ð ð ð ð"
ð 
ð 
ð'ð 'ð 'ð6Fð Fð Fð Fð Fr   r†   c                  ó\   — e Zd ZdZddddœdd„Zd„ Zd	„ Zd
„ Zej	        d„ ¦   «         Z
d„ ZdS )ÚRowwiseParalleluL  Row-wise: weight â†’ Shard(1), bias â†’ Replicate (Embedding: weight â†’ Shard(0)).

    Linear input is sharded on the last dim; Embedding input is replicated. The module
    forward produces a Partial output which the boundary redistribute reduces to
    output_layouts (Replicate â†’ allreduce, Shard(1) â†’ reduce-scatter).
    NTr‡   rŠ   r‹   c               ój   — |pt          d¦  «        | _        |pt          ¦   «         | _        || _        d S r�   )r   rˆ   r   r‰   rŠ   r�   s       r   r�   zRowwiseParallel.__init__ü   s4   € Ø*Ð7­e°B©i¬iˆÔØ,Ð;µ	±´ˆÔØ 0ˆÔÐÐr   c                ó’   — t          |dd¦  «        }t          |t          j        j        ¦  «        ot          j        ¦   «          }|p|S r’   r”   r˜   s       r   re   z(RowwiseParallel.should_use_local_tensors  r›   r   c                óf  — |j                              |¦  «        }|€d S t          |t          j        j        ¦  «        rt          d¦  «        }n#|dk    rt          ¦   «         nt          d¦  «        }t          j                             t          |||gd ¬¦  «        |j
        ¬¦  «        |j         |<   d S )Nr   ÚbiasrŽ   rª   rM   )rQ   r�   rC   rV   rW   rž   r   r   rX   r   rN   r¬   s         r   rk   zRowwiseParallel.shard_param  sª   € ØÔ!×%Ò% eÑ,Ô,ˆØˆ<ØˆFÝ�f�eœhÔ0Ñ1Ô1ð 	FÝ˜a™œˆIˆIð (-°¢ �	™œ˜½EÀ"¹I¼IˆIÝ$)¤H×$6Ò$6Ý˜d D¨9¨+ÀTÐJÑJÔJØÔ,ð %7ñ %
ô %
ˆÔ˜5Ñ!Ð!Ð!r   c                ó  — t          |t          j        j        ¦  «        rt	          ¦   «         nt          d¦  «        }|d         }| j        |k    }|                      |¦  «        r|rt          |t          ¦  «        s||fS t          |t          ¦  «        st          j	        ||| j        gd¬¦  «        }|j
        |fk    r|                     |g¬¦  «        }|                      |¦  «        r|                     ¦   «         }|f|dd …         z   |fS )NrŽ   r   FrK   r°   r   )rC   rV   rW   rž   r   r   rˆ   re   r	   rS   rU   r¶   rJ   )rd   rY   rn   ro   rg   Údesiredr·   Úinput_has_desired_layouts           r   rp   z,RowwiseParallel.transform_inputs_pre_forward  s  € å!+¨FµE´HÔ4FÑ!GÔ!GÐV•)‘+”+�+ÍUÐSUÉYÌYˆØ�ŒGˆð $(Ô#5¸Ò#@Ð Ø×(Ò(¨Ñ0Ô0ð 	 Ð5Mð 	 ÕV`ÐabÕdkÑVlÔVlð 	 Ø˜�<ÐÝ˜!�WÑ%Ô%ð 	SÝÔ" 1 d¨TÔ-?Ð,@ÈEÐRÑRÔRˆAØŒ<˜G˜:Ò%Ð%Ø—’¨7¨)�Ñ4Ô4ˆAØ×(Ò(¨Ñ0Ô0ð 	Ø—
’
‘”ˆAØˆt�d˜1˜2˜2”h‰ Ð&Ð&r   c              #  ó"  K  — |                       |¦  «        sd V — d S |j                             d¦  «        }|�
d |j        d<   	 t          |¦  «        5  d V — d d d ¦  «         n# 1 swxY w Y   |�||j        d<   d S d S # |�
||j        d<   w xY w)NrÄ   )re   rQ   r�   r_   )rd   rY   rg   rÄ   s       r   ru   z&RowwiseParallel.context_around_forward%  s  è è € à×,Ò,¨VÑ4Ô4ð 	6ØˆEˆEˆEˆEˆEð
 Ô%×)Ò)¨&Ñ1Ô1ˆDØÐØ-1�Ô" 6Ñ*ð6Ý.¨vÑ6Ô6ð ð Ø�E�E�Eðð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð ð Ð#Ø15�FÔ& vÑ.Ð.Ð.ð $Ð#ø�4Ð#Ø15�FÔ& vÑ.Ð5Ð5Ð5Ð5s0   ÁB  ÁA%ÁB  Á%A)Á)B  Á,A)Á-B  Â Bc                óR  — t          |t          j        j        ¦  «        o7t          |t          ¦  «         o!t          | j        t          ¦  «        o|j         }|r•|j        dk    r| 	                    ¦   «         n| 	                    d¦  «        }t          j        ||¬¦  «         |j                             d¦  «        x}�.|t          |t          ¦  «        r|                     ¦   «         n|z   }n¸t          |t          ¦  «        s%t	          j        ||t!          ¦   «         gd¬¦  «        }|j        | j        fk    r|                     | j        g¬¦  «        }|                      |¦  «        r!|j                             d¦  «        x}�||z   }| j        r|                     ¦   «         }|S )Nr   r¯   r   rÄ   FrK   r°   )rC   rV   rW   r–   r	   r‰   r   rN   rŸ   r³   ÚdistÚ
all_reducerQ   r�   rJ   rS   r
   rU   r¶   re   rŠ   )rd   rY   rx   rg   Úuse_local_inference_pathr¹   rÄ   s          r   ry   z-RowwiseParallel.transform_output_post_forward7  sž  € å�v�uœxœÑ/Ô/ð )Ý˜v¥wÑ/Ô/Ð/ð)å˜4Ô.µ	Ñ:Ô:ð)ð Ô(Ð(ð	 	!ð $ð 	+Ø04´	¸Q²°˜DŸNšNÑ,Ô,Ð,ÀDÇNÂNÐSWÑDXÔDXˆMÝŒO˜F¨-Ð8Ñ8Ô8Ð8ØÔ*×.Ò.¨vÑ6Ô6Ð6�ÐCØµjÀÅwÑ6OÔ6OÐ#Y 4§=¢=¡?¤? ?ÐUYÑZ�øõ ˜f¥gÑ.Ô.ð XÝ Ô+¨F°D½7¹9¼9¸+ÐQVÐWÑWÔW�ØÔ  TÔ%8Ð$:Ò:Ð:Ø×,Ò,¸Ô9LÐ8MÐ,ÑNÔN�Ø×,Ò,¨VÑ4Ô4ð 'À&ÔBT×BXÒBXÐY_ÑB`ÔB`Ð:`¸$Ð9mØ $™�ØÔ$ð +ØŸšÑ*Ô*�àˆr   r½   )r‚   rƒ   r„   r¾   r�   re   rk   rp   rr   Úcontextmanagerru   ry   r$   r   r   rÀ   rÀ   ô   s¤   € € € € € ðð ð )-¸TÐ\`ð 1ð 1ð 1ð 1ð 1ð 1ð
Gð Gð Gð

ð 
ð 
ð'ð 'ð 'ð" Ôð6ð 6ñ Ôð6ð"ð ð ð ð r   rÀ   c                  ó   — e Zd ZdZd„ ZdS )ÚReplicatedWithGradAllReduceuÉ  Replicated parameter whose gradient is partial.

    For norms that sit between a colwise and a rowwise layer and normalize along a sharded
    axis â€” e.g. Qwen3's per-head ``q_norm``/``k_norm``, which only see this rank's heads. The
    forward needs no collective (the param is replicated and the activation is already local),
    but each rank's parameter gradient only covers its own heads, so the gradients have to be
    summed across the mesh.
    c                ó<   ‡— ˆfd„}|                      |¦  «         |S )Nc                ó¢   •— |                       d¬¦  «        D ]6}|j        �-t          j        |j        ‰                     ¦   «         ¬¦  «         Œ7d S )NFrG   r   )Ú
parametersÚgradrÊ   rË   r³   )ÚmodÚ
grad_inputÚgrad_outputrE   rg   s       €r   Ú_all_reduce_gradszFReplicatedWithGradAllReduce.install_forward.<locals>._all_reduce_grads^  sX   ø€ ØŸš°˜Ñ6Ô6ð Hð H�Ø”:Ð)Ý”O E¤J°d·n²nÑ6FÔ6FÐGÑGÔGÐGøðHð Hr   )Úregister_full_backward_hook)rd   rY   rg   r×   s     ` r   r�   z+ReplicatedWithGradAllReduce.install_forward[  s=   ø€ ð	Hð 	Hð 	Hð 	Hð 	Hð
 	×*Ò*Ð+<Ñ=Ô=Ð=Øˆr   N)r‚   rƒ   r„   r¾   r�   r$   r   r   rÏ   rÏ   Q  s-   € € € € € ðð ð	ð 	ð 	ð 	ð 	r   rÏ   c                  ó   — e Zd ZdZd„ ZdS )ÚAllReduceParallelz@All-reduce a module's partial forward output across the TP mesh.c                ó  — |€d S t          |t          ¦  «        s%t          j        ||t          ¦   «         gd¬¦  «        }|j        t          ¦   «         fk    r#|                     t          ¦   «         g¬¦  «        }|                     ¦   «         S )NFrK   r°   )rC   r	   rS   r
   rU   r   r¶   rJ   rw   s       r   ry   z/AllReduceParallel.transform_output_post_forwardj  s~   € Øˆ>Ø�4Ý˜&¥'Ñ*Ô*ð 	TÝÔ'¨°µw±y´y°kÈUÐSÑSÔSˆFØÔ¥¡¤ Ò.Ð.Ø×(Ò(µY±[´[°MÐ(ÑBÔBˆFØ�ŠÑ Ô Ð r   N©r‚   rƒ   r„   r¾   ry   r$   r   r   rÚ   rÚ   g  s)   € € € € € ØJÐJð!ð !ð !ð !ð !r   rÚ   c                  ó   — e Zd ZdZd„ ZdS )ÚMlaKvAProjParallela#  
    For MLA attention used in DeepSeek-V2 style models (deepseek_v2, longcat_flash, glm_moe_dsa, glm4_moe_lite):
    kv_a_proj_with_mqa output is [kv_lora_rank + qk_rope_head_dim] (can have different naming but important thing
    to understand is that it is split)
    Example below (from modeling_longcat_flash.py):

    kv_a_proj_with_mqa
            |
            split
            /            k_pass    k_rot  <-- "bypasses kv_b_proj"
        |          |        (goes straight to attention,
    kv_a_layernorm |         never touches kv_b_proj)
        |          |
    kv_b_proj      |
    (colwise)      |
        |          |
        k_pass     k_rot
            \      /
               cat
                |
            key_states

    k_pass is passed to kv_b_proj (colwise) which has built-in all_reduce_backward so we don't have a partial gradient for it.
    However, k_rot goes straight to attention, never touches kv_b_proj. So we need to average gradient across all ranks otherwise we only get gradient for one rank (partial gradient).
    c                ól  — t          |j        d¦  «        s%t          dt          |¦  «        j        › d�¦  «        ‚|j        j        }|                     |j        d         |z
  |gd¬¦  «        \  }}t           	                    || 
                    ¦   «         ¦  «        }t          j        ||gd¬¦  «        S )NÚqk_rope_head_dimzConfig for z· does not have `qk_rope_head_dim`. MlaKvAProjParallel requires `qk_rope_head_dim` to be defined in the model config. Please add it to the model's config or update the TP plan mapping.rŽ   )r»   )ÚhasattrÚconfigÚAttributeErrorÚtyper‚   rà   Úsplitr    r´   rµ   r³   rV   Úcat)rd   rY   rx   rg   Úrope_dimÚpass_outputÚrope_outputs          r   ry   z0MlaKvAProjParallel.transform_output_post_forward�  s½   € Ý�v”}Ð&8Ñ9Ô9ð 	Ý ðU�d 6™lœlÔ3ð Uð Uð Uñô ð ð
 ”=Ô1ˆØ#)§<¢<°´¸bÔ1AÀHÑ1LÈhÐ0WÐ]_ <Ñ#`Ô#`Ñ ˆ�[Ý(×.Ò.¨{¸D¿NºNÑ<LÔ<LÑMÔMˆÝŒy˜+ {Ð3¸Ð<Ñ<Ô<Ð<r   NrÜ   r$   r   r   rÞ   rÞ   t  s-   € € € € € ðð ð6
=ð 
=ð 
=ð 
=ð 
=r   rÞ   c                  ó:   ‡ — e Zd Zdddœdd„Zˆ fd	„Zd
„ Zd„ Zˆ xZS )ÚSequenceParallelr   T©Úsequence_dimrŠ   rí   ÚintrŠ   r‹   c               ó"   — || _         || _        d S rm   rì   )rd   rí   rŠ   s      r   r�   zSequenceParallel.__init__ž  s   € Ø(ˆÔØ 0ˆÔÐÐr   c                óF  •— t          |                     d¬¦  «        ¦  «        D ][\  }}|                     |t          j                             t          j        ||t          ¦   «         gd¬¦  «        ¦  «        ¦  «         Œ\t          ¦   «          
                    ||¦  «        S )NFrG   rK   )ÚlistrO   Úregister_parameterrV   rW   rX   r	   rS   r   Úsuperr�   )rd   rY   rg   Úp_nameÚpÚ	__class__s        €r   r�   z SequenceParallel.install_forward¢  s˜   ø€ å˜f×5Ò5¸eÐ5ÑDÔDÑEÔEð 	ð 	‰IˆF�AØ×%Ò%Ø�œ×*Ò*­7Ô+=¸aÀÍ	ÉÌÀ}Ð`eÐ+fÑ+fÔ+fÑgÔgñô ð ð õ ‰wŒw×&Ò& v¨tÑ4Ô4Ð4r   c                óþ   — t          | j        ¦  «        }|d         }t          |t          ¦  «        st          j        |||gd¬¦  «        }n#|j        |fk    r|                     |g¬¦  «        }|f|dd …         z   |fS )Nr   FrK   r°   r   )r   rí   rC   r	   rS   rU   r¶   )rd   rY   rn   ro   rg   Úseqr·   s          r   rp   z-SequenceParallel.transform_inputs_pre_forwardª  s‡   € Ý�DÔ%Ñ&Ô&ˆØ�ŒGˆÝ˜!�WÑ%Ô%ð 	1ÝÔ" 1 d¨S¨E¸UÐCÑCÔCˆAˆAØŒ\˜c˜VÒ#Ð#Ø—’¨3¨%�Ñ0Ô0ˆAØˆt�d˜1˜2˜2”h‰ Ð&Ð&r   c                ój   — t          |t          ¦  «        r| j        r|                     ¦   «         n|S |S rm   )rC   r	   rŠ   rJ   rw   s       r   ry   z.SequenceParallel.transform_output_post_forward³  s7   € Ý�f�gÑ&Ô&ð 	JØ(,Ô(=ÐI�6—?’?Ñ$Ô$Ð$À6ÐIØˆr   )rí   rî   rŠ   r‹   )r‚   rƒ   r„   r�   r�   rp   ry   Ú__classcell__)rö   s   @r   rë   rë   �  sy   ø€ € € € € Ø./È$ð 1ð 1ð 1ð 1ð 1ð 1ð5ð 5ð 5ð 5ð 5ð'ð 'ð 'ðð ð ð ð ð ð r   rë   c                  óB   — e Zd ZdZdddœdd	„Zd
„ Zdd„Zd„ Zd„ Zd„ Z	dS )ÚPackedColwiseParallelzVColumn-wise parallel style for fused linear weights packed along the output dimension.Tr   )rŠ   Úsplit_factorrŠ   r‹   rý   rî   c               ór   — t          ¦   «         f| _        || _        || _        t	          ¦   «         g| _        d S rm   )r   rˆ   rŠ   rý   r
   Úinput_grad_placements)rd   rŠ   rý   s      r   r�   zPackedColwiseParallel.__init__¼  s8   € õ (™kœk˜^ˆÔØ 0ˆÔØ(ˆÔõ '.¡i¤i [ˆÔ"Ð"Ð"r   c                ó   — dS ©NTr$   rc   s     r   re   z.PackedColwiseParallel.should_use_local_tensorsÉ  ó   € Øˆtr   Ú
param_ndimr   c                ó   — |dk    rdS |dz
  S )zYDimension holding packed gate/up features: dim 0 for 2D Linear, dim 1 for 3D MoE experts.r   rŽ   r   r$   )rd   r  s     r   Ú_packed_output_shard_dimz.PackedColwiseParallel._packed_output_shard_dimÌ  s   € à˜Š?ˆ?Ø�2Ø˜A‰~Ðr   c                óX  — |j                              |¦  «        }|€d S |                      |j        ¦  «        }|j        dk    rt	          |¦  «        }nt          || j        ¬¦  «        }t          j         	                    t          |||gd ¬¦  «        |j        ¬¦  «        |j         |<   d S )Nr   ©r»   rý   rª   rM   )rQ   r�   r  rŸ   r   r   rý   rV   rW   rX   r   rN   )rd   rY   rE   rg   r£   r¥   r­   s          r   rk   z!PackedColwiseParallel.shard_paramÒ  s¬   € ØÔ!×%Ò% eÑ,Ô,ˆØˆ<ØˆFØ×1Ò1°$´)Ñ<Ô<ˆ	àŒ9˜Š>ˆ>Ý˜iÑ(Ô(ˆIˆIå%¨)À$ÔBSÐTÑTÔTˆIÝ$)¤H×$6Ò$6Ý˜d D¨9¨+ÀTÐJÑJÔJØÔ,ð %7ñ %
ô %
ˆÔ˜5Ñ!Ð!Ð!r   c                ó4  — |d         }t          |t          ¦  «        st          j        ||| j        d¬¦  «        }n+|j        | j        k    r|                     | j        ¬¦  «        }|                     t          ¦   «         g¬¦  «        }|f|dd …         z   |fS )Nr   FrK   r°   r±   r   )rC   r	   rS   rˆ   rU   r¶   rJ   r
   ©rd   rY   rn   ro   rg   Úinput_tensors         r   rp   z2PackedColwiseParallel.transform_inputs_pre_forwardá  sŸ   € Ø˜A”wˆå˜,­Ñ0Ô0ð 	TÝ"Ô-¨l¸DÀ$ÔBTÐ`eÐfÑfÔfˆLˆLØÔ$¨Ô(:Ò:Ð:Ø'×4Ò4ÀÔ@RÐ4ÑSÔSˆLð
 $×,Ò,½g¹i¼i¸[Ð,ÑIÔIˆØˆ  a b b¤Ñ)¨6Ð1Ð1r   c                ór   — |�| j         r|S t          j        ||t          d| j        ¬¦  «        fd¬¦  «        S )NrŽ   r  FrK   )rŠ   r	   rS   r   rý   rw   s       r   ry   z3PackedColwiseParallel.transform_output_post_forwardï  sJ   € Øˆ>˜TÔ2ˆ>ØˆMÝÔ!Ø�D�=¨R¸dÔ>OÐPÑPÔPÐRÐ^cð
ñ 
ô 
ð 	
r   N)rŠ   r‹   rý   rî   )r  rî   r   rî   )
r‚   rƒ   r„   r¾   r�   re   r  rk   rp   ry   r$   r   r   rü   rü   ¹  s�   € € € € € Ø`Ð`ð
 "&Øð	1ð 1ð 1ð 1ð 1ð 1ðð ð ðð ð ð ð
ð 
ð 
ð2ð 2ð 2ð
ð 
ð 
ð 
ð 
r   rü   c                  ó&   — e Zd ZdZddœd	d„Zd„ ZdS )
ÚPackedRowwiseParallelzCParameter style for fused weights packed along the final dimension.r   ©rý   rý   rî   c               ó   — || _         d S rm   r  )rd   rý   s     r   r�   zPackedRowwiseParallel.__init__ú  s   € Ø(ˆÔÐÐr   c                ó   — |j                              |¦  «        }|€d S |j        dk    rt          ¦   «         nt	          d| j        ¬¦  «        }t          j                             t          |||gd ¬¦  «        |j
        ¬¦  «        |j         |<   d S )Nr   rŽ   r  rª   rM   )rQ   r�   rŸ   r   r   rý   rV   rW   rX   r   rN   r¬   s         r   rk   z!PackedRowwiseParallel.shard_paramý  s�   € ØÔ!×%Ò% eÑ,Ô,ˆØˆ<ØˆFØ#'¤9°¢> >•I‘K”K�Kµ}ÈÐZ^ÔZkÐ7lÑ7lÔ7lˆ	Ý$)¤H×$6Ò$6Ý˜d D¨9¨+ÀTÐJÑJÔJØÔ,ð %7ñ %
ô %
ˆÔ˜5Ñ!Ð!Ð!r   N)rý   rî   ©r‚   rƒ   r„   r¾   r�   rk   r$   r   r   r  r  ÷  sJ   € € € € € ØMÐMà./ð )ð )ð )ð )ð )ð )ð
ð 
ð 
ð 
ð 
r   r  c                  ó&   — e Zd ZdZddœd	d„Zd„ ZdS )
ÚMoEParamShardzÉParam-only EP style for MoE expert weights (``grouped_gemm``).

    Shards dim 0 and updates module.num_experts to the per-rank local count so the
    experts forward and ep_router sentinel agree.
    F©Úshards_expert_dimr  r‹   c               ó"   — || _         || _        d S rm   )r­   r  )rd   r­   r  s      r   r�   zMoEParamShard.__init__  s   € Ø"ˆŒØ!2ˆÔÐÐr   c                óŽ  — |j                              |¦  «        }|€d S | j        rZt          |d¦  «        rJ|j        d         }|                     ¦   «         }||z  dk    rt          d|› d|› d�¦  «        ‚||z  |_        t          j	         
                    t          ||| j        gd ¬¦  «        |j        ¬¦  «        |j         |<   d S )NÚnum_expertsr   zCannot evenly shard z experts across z expert-parallel ranks.rª   rM   )rQ   r�   r  rá   r    r¡   r¢   r  rV   rW   rX   r   r­   rN   )rd   rY   rE   rg   r£   Úglobal_num_expertsÚexpert_parallel_sizes          r   rk   zMoEParamShard.shard_param  s   € ØÔ!×%Ò% eÑ,Ô,ˆØˆ<ØˆFØÔ!ð 	L¥g¨f°mÑ&DÔ&Dð 	LØ!%¤¨A¤ÐØ#'§9¢9¡;¤;Ð Ø!Ð$8Ñ8¸AÒ=Ð=Ý ðEÐ+=ð Eð EØ+ðEð Eð Eñô ð ð "4Ð7KÑ!KˆFÔÝ$)¤H×$6Ò$6Ý˜d D¨4¬>Ð*:È$ÐOÑOÔOØÔ,ð %7ñ %
ô %
ˆÔ˜5Ñ!Ð!Ð!r   N)r  r‹   r  r$   r   r   r  r    sR   € € € € € ðð ð @Eð 3ð 3ð 3ð 3ð 3ð 3ð
ð 
ð 
ð 
ð 
r   r  c                  ó>   — e Zd ZdZed„ ¦   «         Zed„ ¦   «         ZdS )Ú_AllReduceForwardz)Allreduce-sum forward, identity backward.c                ób   — t          j        |¦  «        dk    rt          j        ||¬¦  «         |S )Nr   r   )rÊ   Úget_world_sizerË   ©Úctxr·   r¹   s      r   r€   z_AllReduceForward.forward+  s2   € åÔ" =Ñ1Ô1°AÒ5Ð5Ý” ¨Ð7Ñ7Ô7Ð7ØˆHr   c                ó
   — |d fS rm   r$   ©r   rÓ   s     r   Úbackwardz_AllReduceForward.backward1  s   € à˜�:Ðr   N©r‚   rƒ   r„   r¾   Ústaticmethodr€   r#  r$   r   r   r  r  (  sN   € € € € € Ø7Ð7à	ð	ð 	ñ 
Œð	ð
 
ð	ð 	ñ 
Œð	ð 	ð 	r   r  c                  ó>   — e Zd ZdZed„ ¦   «         Zed„ ¦   «         ZdS )r´   a~  Identity forward, allreduce-sum backward.

        Used for MoE routing weights: the forward value is replicated (same on all
        ranks), but the backward gradient is partial (each rank has 1/tp_size from
        its expert shard). We need to sum the partial gradients without dividing by
        world_size, which is what DTensor's Replicate backward does incorrectly.
        c                ó   — || _         |S rm   )r¹   r  s      r   r€   z_AllReduceBackward.forward>  s   € à -ˆCÔØˆHr   c                óh   — |                      ¦   «         }t          j        || j        ¬¦  «         |d fS )Nr   )Ú
contiguousrÊ   rË   r¹   r"  s     r   r#  z_AllReduceBackward.backwardC  s3   € à—?’?Ñ$Ô$ˆDÝŒO˜D¨Ô(9Ð:Ñ:Ô:Ð:Ø˜�:Ðr   Nr$  r$   r   r   r´   r´   5  sT   € € € € € ð	ð 	ð 
ð	ð 	ñ 
Œð	ð 
ð	ð 	ñ 
Œð	ð 	ð 	r   r´   c                  ó4   — e Zd Zd„ Zddœd„Zddœd„Zdd„ZdS )	ÚMoeExpertsParallelc                ó   — dS r  r$   rc   s     r   re   z+MoeExpertsParallel.should_use_local_tensorsK  r  r   F©Úis_expert_parallelc               óÒ  — |^}}|j         dk    r|                     ¦   «         n|                     d¦  «        }t          |t          ¦  «        r|                     ¦   «         }t
                               ||¦  «        }t          |¦  «        dk    rQ|^}	}
}t          |
t          ¦  «        r|
                     ¦   «         }
|st
                               |
|¦  «        }
|	|
g|¢}|g|¢R |fS )Nr   r¯   r   )rŸ   r³   rC   r	   rJ   r´   rµ   r/   )rd   rY   rn   ro   rg   r.  Úhidden_statesÚrouting_argsÚtp_groupÚtop_k_indexÚtop_k_weightsÚ
extra_argss               r   rp   z/MoeExpertsParallel.transform_inputs_pre_forwardN  só   € Ø'+Ð$ˆ˜Ø'+¤y°A¢~ ~�4—>’>Ñ#Ô#Ð#¸4¿>º>È$Ñ;OÔ;OˆÝ�m¥WÑ-Ô-ð 	5Ø)×2Ò2Ñ4Ô4ˆMÝ*×0Ò0°ÀÑIÔIˆåˆ|ÑÔ Ò!Ð!Ø6BÐ3ˆK˜¨Ý˜-­Ñ1Ô1ð 9Ø -× 6Ò 6Ñ 8Ô 8�Ø%ð RÝ 2× 8Ò 8¸ÈÑ QÔ Q�Ø'¨ÐD¸ÐDˆLàÐ- Ð-Ð-¨vÐ5Ð5r   c               óÐ   ‡ ‡‡‡‡‡— ‰j         Št          d„ ‰                     ¦   «         D ¦   «         ¦  «        rt          ¦   «         nt	          ¦   «         Šˆˆˆˆˆˆ fd„}|‰_         ‰S )zIInstall the transforms but pass `is_expert_parallel` in the forward call.c              3  ó|   K  — | ]7}t          |t          ¦  «        ot          d „ |j        D ¦   «         ¦  «        V — Œ8dS )c              3  ó@   K  — | ]}|                      ¦   «          V — Œd S rm   ©Úis_replicate©r%   r­   s     r   ú	<genexpr>z?MoeExpertsParallel.install_forward.<locals>.<genexpr>.<genexpr>e  s2   è è € Ð2rÐ2rÐT]°y×7MÒ7MÑ7OÔ7OÐ3OÐ2rÐ2rÐ2rÐ2rÐ2rÐ2rr   N©rC   r	   ÚanyrU   ©r%   rE   s     r   r<  z5MoeExpertsParallel.install_forward.<locals>.<genexpr>d  s`   è è € ð ð àõ ˜5¥'Ñ*Ô*Ðr­sÐ2rÐ2rÐafÔaqÐ2rÑ2rÔ2rÑ/rÔ/rðð ð ð ð ð r   c                 óÜ   •— ‰                      ‰| |‰‰¬¦  «        \  } }‰                     ‰‰¦  «        5   ‰| i |¤Ž}d d d ¦  «         n# 1 swxY w Y   ‰                     ‰|‰‰¬¦  «        S )Nr-  )Úsourcer}   )	rn   ro   rx   r.  rg   rY   r~   Úoutput_sourcerd   s	      €€€€€€r   r   z6MoeExpertsParallel.install_forward.<locals>.tp_forwardk  sÍ   ø€ Ø×<Ò<Ø˜˜f dÐ?Qð =ñ ô ‰LˆD�&ð ×,Ò,¨V°TÑ:Ô:ð ;ð ;Ø)Ð)¨4Ð:°6Ð:Ð:�ð;ð ;ð ;ñ ;ô ;ð ;ð ;ð ;ð ;ð ;ð ;øøøð ;ð ;ð ;ð ;à×5Ò5°f¸fÀdÐS`Ð5ÑaÔaÐas   µ	A
Á
AÁA)r€   r>  rÒ   r
   r   )rd   rY   rg   r.  r   r~   rB  s   ```` @@r   r�   z"MoeExpertsParallel.install_forward_  s¬   øøøøøø€ à!œ>Ðõ ð ð à#×.Ò.Ñ0Ô0ðñ ô ñ ô ð�G‰IŒIˆIõ
 ‘”ð 	ð	bð 	bð 	bð 	bð 	bð 	bð 	bð 	bð 	bð 	bð $ˆŒØˆr   Nc                ó  — |€d S t          d„ |                     ¦   «         D ¦   «         ¦  «        }|s|S |j        dk    r|                     ¦   «         n|                     d¦  «        }t                               ||¦  «        S )Nc              3  ó|   K  — | ]7}t          |t          ¦  «        ot          d „ |j        D ¦   «         ¦  «        V — Œ8dS )c              3  ó@   K  — | ]}|                      ¦   «          V — Œd S rm   r9  r;  s     r   r<  zMMoeExpertsParallel.transform_output_post_forward.<locals>.<genexpr>.<genexpr>{  s2   è è € Ð.nÐ.nÐPY°9×3IÒ3IÑ3KÔ3KÐ/KÐ.nÐ.nÐ.nÐ.nÐ.nÐ.nr   Nr=  r?  s     r   r<  zCMoeExpertsParallel.transform_output_post_forward.<locals>.<genexpr>z  s`   è è € ð %
ð %
àõ �u�gÑ&Ô&Ðn­3Ð.nÐ.nÐ]bÔ]mÐ.nÑ.nÔ.nÑ+nÔ+nð%
ð %
ð %
ð %
ð %
ð %
r   r   r¯   )r>  rÒ   rŸ   r³   r  rµ   )rd   rY   rx   rg   rA  Úhas_sharded_parametersr¹   s          r   ry   z0MoeExpertsParallel.transform_output_post_forwardv  s“   € Øˆ>Ø�4å!$ð %
ð %
à×*Ò*Ñ,Ô,ð%
ñ %
ô %
ñ "
ô "
Ðð &ð 	ØˆMà,0¬I¸ªN¨N˜ŸšÑ(Ô(Ð(ÀÇÂÈtÑ@TÔ@TˆÝ ×&Ò& v¨}Ñ=Ô=Ð=r   rm   )r‚   rƒ   r„   re   rp   r�   ry   r$   r   r   r+  r+  J  sw   € € € € € ðð ð ð ^cð 6ð 6ð 6ð 6ð 6ð" CHð ð ð ð ð ð.>ð >ð >ð >ð >ð >r   r+  c                  ó   — e Zd ZdZd„ ZdS )ÚMoeIdentityParallelzé
    Used in longcat_flash zero_experts (nn.Identity) which return the same value on every GPU, but the
    moe_tp_experts will sums across GPUs. Therefore we pre-divide the identity input by tp_size to cancel the extra scaling.
    c                ó\   — |d         }||                      ¦   «         z  g|dd …         ¢R |fS )Nr   r   )r¡   r	  s         r   rp   z0MoeIdentityParallel.transform_inputs_pre_forward‹  s6   € Ø˜A”wˆØ˜tŸyšy™{œ{Ñ*Ð6¨T°!°"°"¬XÐ6Ð6¸Ð>Ð>r   N)r‚   rƒ   r„   r¾   rp   r$   r   r   rH  rH  …  s-   € € € € € ðð ð
?ð ?ð ?ð ?ð ?r   rH  c                  ó   — e Zd ZdZd„ ZdS )ÚEpRouterParallelu¡  Expert-parallel router: forward-only slicing of router outputs to local experts.

    Expects the router to return `(router_logits, router_scores, router_indices, *extra)`.
    `router_logits` and any trailing `extra` outputs are passed through unchanged.

    The gate runs replicated on every rank and emits global expert IDs and scores.
    Under EP each rank owns `num_experts // ep_size` experts, so this post-forward hook:

    - zeroes scores for non-local experts
    - remaps surviving global indices to local indices (`fmod` after masking non-local slots)
    - sets dropped slots to sentinel `num_local_experts` (skipped by grouped_gemm experts forward)

    Downstream `moe_tp_experts` allreduce-sums partial per-rank expert outputs.

    Example: 4 tokens, top_k=4, 128 experts, EP=8 â†’ num_local_experts=16.

    Router output (identical on all ranks):
        router_indices:
            [ 52,  42, 119,  67],
            [102,  89,  61,  40],
            [ 82, 103,   4,  34],
            [ 93,  23, 109,  11]

    Owning rank (`index // 16`):
            [ 3,  2,  7,  4],
            [ 6,  5,  3,  2],
            [ 5,  6,  0,  2],
            [ 5,  1,  6,  0],

    After slicing on rank 0 (owns experts 0-15):
        router_indices:          router_scores (illustrative):
            [16, 16, 16, 16],       [0.0, 0.0, 0.0, 0.0],
            [16, 16, 16, 16],       [0.0, 0.0, 0.0, 0.0],
            [16, 16,  4, 16],       [0.0, 0.0, 0.3, 0.0],
            [16, 16, 16, 11],       [0.0, 0.0, 0.0, 0.1],

    On rank 1 (owns experts 16-31), global expert 23 remaps to local index 7 via `23 % 16`.

    Scores and indices stay paired element-wise in `(seq, top_k)` shape.
    c                óv  — |                      ¦   «         |                     ¦   «         }}t          |dd ¦  «        }|€ t          t          |dd ¦  «        dd ¦  «        }|€%t          dt	          |¦  «        j        › d�¦  «        ‚||z  dk    rt          d|› d|› d�¦  «        ‚||z  }|^}}	}
}t          j        ¦   «         rV|	j	        rO|j
        d	k    r|                     ¦   «         n|                     d
¦  «        }t                               |	|¦  «        }	|
|z  |k    }|	                     |d¦  «        }	|
                     |d¦  «        }
|d	k    rt          j        |
|¦  «        }
n2|
                     |
dk    d¦  «                             |
dk     d¦  «        }
|
                     |
dk    |¦  «        }
||	|
g|¢R S )Nr  râ   zRouter module z2 is missing `num_experts` and `config.num_experts`r   z*num_experts must be divisible by ep_size: z % z != 0r   r¯   g        rŽ   )Úget_local_rankr¡   r•   rã   rä   r‚   r¢   rV   r—   rN   rŸ   r³   r´   rµ   Úmasked_fillÚfmod)rd   rY   rx   rg   Úep_rankÚep_sizer  Únum_local_expertsÚrouter_logitsÚrouter_scoresÚrouter_indicesÚextra_outputsr¹   Únon_local_masks                 r   ry   z.EpRouterParallel.transform_output_post_forwardº  së  € Ø×.Ò.Ñ0Ô0°$·)²)±+´+�ˆÝ˜f m°TÑ:Ô:ˆØÐÝ!¥'¨&°(¸DÑ"AÔ"AÀ=ÐRVÑWÔWˆKØÐÝ Øj¥ f¡¤Ô!6ÐjÐjÐjñô ð ð ˜Ñ  AÒ%Ð%ÝÐhÈ+ÐhÐhÐZaÐhÐhÐhÑiÔiÐiØ'¨7Ñ2ÐàGMÐDˆ�} n°}õ Ô Ñ"Ô"ð 	S }Ô'Bð 	SØ04´	¸Q²°˜DŸNšNÑ,Ô,Ð,ÀDÇNÂNÐSWÑDXÔDXˆMÝ.×4Ò4°]ÀMÑRÔRˆMØ(Ð,=Ñ=À'ÒIˆØ%×1Ò1°.À#ÑFÔFˆØ'×3Ò3°NÀBÑGÔGˆØ˜qÒ Ð Ý"œZ¨Ð8IÑJÔJˆNˆNà+×7Ò7¸ÈÒ8JÈAÑNÔN×ZÒZÐ[iÐlmÒ[mÐoqÑrÔrˆNØ'×3Ò3°NÀbÒ4HÐJ[Ñ\Ô\ˆØ˜m¨^ÐK¸mÐKÐKÐKr   NrÜ   r$   r   r   rK  rK  �  s3   € € € € € ð'ð 'ðRLð Lð Lð Lð Lr   rK  c                  ó   — e Zd ZdZd„ ZdS )ÚRouterParallelMegaMoez÷Router TP plan used with DeepGEMM Mega MoE.

    Mega MoE handles EP dispatch inside the kernel and wants raw global expert ids
    with unmasked routing weights, so the router doesn't pre-shard per EP rank like
    ``EpRouterParallel`` does.
    c                ó   — |S rm   r$   rw   s       r   ry   z3RouterParallelMegaMoe.transform_output_post_forwardà  rz   r   NrÜ   r$   r   r   rY  rY  Ø  s-   € € € € € ðð ðð ð ð ð r   rY  c                  ó,   — e Zd ZdZddœd„Zd„ Zdd„ZdS )	ÚMoeTensorParalellMegaMoeExpertsu|  TP layer for DeepGEMM Mega MoE experts.

    Mega MoE is inference-only (the kernel has no backward) and handles EP dispatch +
    combine + per-rank token sharding internally â€” so we skip the gradient-sync hooks
    that ``MoeExpertsParallel`` would apply, and we forward the EP ``process_group``
    into the module so the symm-buffer rendezvous can run on first forward.
    Fr-  c               óf   — |d         |d         |d         }}}||||                      ¦   «         f|fS )Nr   r   r   )r³   )	rd   rY   rn   ro   rg   r.  r0  r3  r4  s	            r   rp   z<MoeTensorParalellMegaMoeExperts.transform_inputs_pre_forwardí  s9   € Ø48¸´G¸TÀ!¼WÀdÈ1Äg M�{ˆØ˜{¨M¸4¿>º>Ñ;KÔ;KÐLÈfÐTÐTr   c                ó    — t          |¦  «        S rm   )r_   rt   s      r   ru   z6MoeTensorParalellMegaMoeExperts.context_around_forwardñ  s   € Ý(¨Ñ0Ô0Ð0r   Nc                ó   — |S rm   r$   )rd   rY   rx   rg   rA  s        r   ry   z=MoeTensorParalellMegaMoeExperts.transform_output_post_forwardô  rz   r   rm   )r‚   rƒ   r„   r¾   rp   ru   ry   r$   r   r   r\  r\  ä  sf   € € € € € ðð ð ^cð Uð Uð Uð Uð Uð1ð 1ð 1ðð ð ð ð ð r   r\  c                  óú  — e Zd ZdZ e¦   «         �rgi d e e¦   «          e¦   «         ¬¦  «        “d e e¦   «          e¦   «         ¬¦  «        “d e e¦   «          e¦   «         ¬¦  «        “d e e¦   «          ed¦  «        ¬¦  «        “d e ed¦  «         e¦   «         ¬¦  «        “d	 e e¦   «          e¦   «         ¬¦  «        “d
 e e¦   «          e¦   «         ¬¦  «        “d e	¦   «         “d e
¦   «         “d ed¬¦  «        “d e ed¦  «        d¬¦  «        “d e¦   «         “d e¦   «         “d e¦   «         “d e¦   «         “d e¦   «         “d e¦   «         “ e¦   «          e¦   «         dœ¥ni ZdS )ÚParallelInterfacez4Registry of named TP styles for the DTensor backend.Úembedding_rowwise)rˆ   r‰   Úcolwise_gather_outputÚcolwise_repÚcolwiserŽ   ÚrowwiseÚrowwise_split_inputÚrowwise_repÚpacked_colwiseÚpacked_rowwiseÚsequence_parallelT)rŠ   Úgrouped_gemmr   r  Ú	ep_routerÚmegamoe_routerÚmoe_tp_expertsÚmegamoe_expertsÚmoe_identity_expertÚreplicated_with_grad_allreduce)Úmla_kv_a_projrË   N)r‚   rƒ   r„   r¾   r   rÀ   r   r†   r   rü   r  rë   r  rK  rY  r+  r\  rH  rÏ   rÞ   rÚ   Ú_global_mappingr$   r   r   ra  ra  ø  s|  € € € € € Ø>Ð>ð0 *Ð)Ñ+Ô+ñ+	ð 	
Ø  ¸y¸y¹{¼{Ð[dÐ[dÑ[fÔ[fÐ!gÑ!gÔ!gð	
à# _ _À9À9Á;Ä;Ð_hÐ_hÑ_jÔ_jÐ%kÑ%kÔ%kð	
ð ˜?˜?¸¸¹¼ÐU^ÐU^ÑU`ÔU`ÐaÑaÔað	
ð ��°Y°Y±[´[ÐQVÐQVÐWYÑQZÔQZÐ[Ñ[Ô[ð		
ð
 ��°U°U¸2±Y´YÈyÈyÉ{Ì{Ð[Ñ[Ô[ð	
ð " ? ?ÀÀÁÄÐ]fÐ]fÑ]hÔ]hÐ#iÑ#iÔ#ið	
ð ˜?˜?¸¸¹¼ÐU^ÐU^ÑU`ÔU`ÐaÑaÔað	
ð Ð3Ð3Ñ5Ô5ð	
ð Ð3Ð3Ñ5Ô5ð	
ð  Ð!1Ð!1À4Ð!HÑ!HÔ!Hð	
ð ˜M˜M¨%¨%°©(¬(ÀdÐKÑKÔKð	
ð Ð)Ð)Ñ+Ô+ð	
ð Ð3Ð3Ñ5Ô5ð	
ð Ð0Ð0Ñ2Ô2ð	
ð Ð>Ð>Ñ@Ô@ð	
ð  "Ð#6Ð#6Ñ#8Ô#8ð!	
ð" -Ð.IÐ.IÑ.KÔ.Kð#	
ð$ 0Ð/Ñ1Ô1Ø+Ð+Ñ-Ô-ð'	
ð 	
ð 	
ð 	
ð, ð/ €O€Or   ra  ÚALL_PARALLEL_STYLESÚNonec           	     ó¼   — d„ | pi                       ¦   «         D ¦   «         }|r9t          d|› dt          t                               ¦   «         ¦  «        › �¦  «        ‚d S )Nc                ó$   — h | ]}|t           v¯|’ŒS r$   )ru  )r%   Ústyles     r   r'   z+_validate_tp_plan_styles.<locals>.<setcomp>  s#   € ÐjÐjÐj EÈÕViÐIiÐIi˜%ÐIiÐIiÐIir   z$Unsupported tensor parallel styles: z. Supported styles are )Úvaluesr¢   rñ   ru  Úkeys)r    Úunsupported_styless     r   Ú_validate_tp_plan_stylesr}    sƒ   € ØjÐj¨g¨m¸×-CÒ-CÑ-EÔ-EÐjÑjÔjÐØð 
ÝðGÐ3Eð Gð GÝ$(Õ)<×)AÒ)AÑ)CÔ)CÑ$DÔ$DðGð Gñ
ô 
ð 	
ð
ð 
r   c                óh  — t          | j        ¦  «         |                      ¦   «         D �]\  }}t          |                     d¬¦  «        ¦  «        D ]o\  }}|r|› d|› �n|}t          || j        d¬¦  «        }|�F|t          v r=t          |         }|                     ||||¬¦  «         |                     |||¦  «         Œpt          || j        d¬¦  «        }|�N|t          v rE|dk    r| j	         
                    ¦   «         |_	        t          |                              ||¦  «         d|_        �Œ	| S )	zKDTensor backend: shard params as placeholders and install TP forward hooks.FrG   r(   T)r:   r    r>   N)r:   rs  )r}  r    Únamed_modulesrñ   rO   r@   ru  rh   rk   râ   Úget_text_configr�   Ú
_is_hooked)	ÚmodelÚtp_meshr   rY   rô   Ú_ÚfullÚ
style_namery  s	            r   Úapply_tensor_parallelismr‡  "  s[  € õ ˜Uœ]Ñ+Ô+Ð+à×+Ò+Ñ-Ô-ð !ñ !‰ˆˆfå˜f×5Ò5¸eÐ5ÑDÔDÑEÔEð 	;ð 	;‰IˆF�AØ)-Ð9�dÐ%Ð%˜VÐ%Ð%Ð%°6ˆDÝ/¸tÈUÌ]ÐfjÐkÑkÔkˆJØÐ%¨*Õ8KÐ*KÐ*KÝ+¨JÔ7�Ø×$Ò$ V¨V°WÈTÐ$ÑRÔRÐRØ×!Ò! &¨&°'Ñ:Ô:Ð:øõ ,¸4ÈÌÐbgÐhÑhÔhˆ
ØÐ! jÕ4GÐ&GÐ&GØ˜_Ò,Ð,ð !&¤× <Ò <Ñ >Ô >�”Ý 
Ô+×;Ò;¸FÀGÑLÔLÐLØ ˆÔÑà€Lr   Ú
state_dictúdict[str, torch.Tensor]Ú_tp_planÚ_tp_sizerî   c                óF  — i }|                       ¦   «         D ]‰\  }}t          |t          j        ¦  «        ret          |t          ¦  «        r|                     ¦   «         }|                     ¦   «                              ¦   «                              ¦   «         ||<   Œ„|||<   ŒŠ|S )z³Gather TP-sharded ``DTensor`` parameters to full CPU tensors for checkpoint saving.

    Every rank must call this function so ``DTensor.full_tensor()`` collectives complete.
    )	rP   rC   rV   ÚTensorr	   Úfull_tensorÚdetachÚcpur)  )rˆ  rŠ  Ú_device_meshr‹  Úgatheredr&   Útensors          r   Úgather_state_dict_for_saver”  >  sš   € ð €HØ!×'Ò'Ñ)Ô)ð #ð #‰ˆˆVÝ�f�eœlÑ+Ô+ð 	#Ý˜&¥'Ñ*Ô*ð .Ø×+Ò+Ñ-Ô-�Ø"ŸMšM™OœO×/Ò/Ñ1Ô1×<Ò<Ñ>Ô>ˆH�S‰MˆMà"ˆH�S‰MˆMØ€Or   )r   r   r   r   )r   r   r    r!   )T)r:   r   r    r;   r   r<   )r    r!   r   rv  )rˆ  r‰  rŠ  r;   r‹  rî   r   r‰  )8Ú
__future__r   rr   r   Úutilsr   Úutils.genericr   Úutils.import_utilsr   r   Ú
get_loggerr‚   r0   rV   Útorch.distributedÚdistributedrÊ   Útorch.distributed.tensorr	   r
   r   r   r   Ú(torch.distributed.tensor.placement_typesr   r   r9   r@   rÍ   r_   ra   r†   rÀ   rÏ   rÚ   rÞ   rë   rü   r  r  ÚautogradÚFunctionr  r´   r+  rH  rK  rY  r\  ra  ru  Ú__annotations__r}  r‡  r”  r$   r   r   ú<module>r¡     s  ðð #Ð "Ð "Ð "Ð "Ð "Ð "à Ð Ð Ð Ø 	€	€	€	à Ð Ð Ð Ð Ð Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø SÐ SÐ SÐ SÐ SÐ SÐ SÐ Sð 
ˆÔ	˜HÑ	%Ô	%€àÐÑÔð Ø€L€L€Là!Ð!Ñ#Ô#ð GØ$Ð$Ð$Ð$Ð$Ð$Ø^Ð^Ð^Ð^Ð^Ð^Ð^Ð^Ð^Ð^Ð^Ð^Ð^Ð^ØFÐFÐFÐFÐFÐFðEð Eð Eð Eð`ð `ð `ð `ð:ð ð ð ð ð$ Ôðð ñ Ôðð<#ð #ð #ð #ð #ñ #ô #ð #ðLPFð PFð PFð PFð PFÐ)ñ PFô PFð PFðfZð Zð Zð Zð ZÐ)ñ Zô Zð Zðzð ð ð ð Ð"5ñ ô ð ð,
!ð 
!ð 
!ð 
!ð 
!Ð+ñ 
!ô 
!ð 
!ð&=ð &=ð &=ð &=ð &=Ð,ñ &=ô &=ð &=ðRð ð ð ð Ð*ñ ô ð ð8;
ð ;
ð ;
ð ;
ð ;
Ð/ñ ;
ô ;
ð ;
ð|
ð 
ð 
ð 
ð 
Ð/ñ 
ô 
ð 
ð"
ð 
ð 
ð 
ð 
Ð'ñ 
ô 
ð 
ð< "Ð!Ñ#Ô#ð !ðð ð ð ð ˜EœNÔ3ñ ô ð ðð ð ð ð ˜Uœ^Ô4ñ ô ð ð*8>ð 8>ð 8>ð 8>ð 8>Ð,ñ 8>ô 8>ð 8>ðv?ð ?ð ?ð ?ð ?Ð-ñ ?ô ?ð ?ðELð ELð ELð ELð ELÐ*ñ ELô ELð ELðP	ð 	ð 	ð 	ð 	Ð,ñ 	ô 	ð 	ðð ð ð ð Ð&8ñ ô ð ð(ð ð ð ð Ð(ñ ô ð ð< *;Ð):Ñ)<Ô)<Ð Ð <Ð <Ð <Ñ <ð
ð 
ð 
ð 
ðð ð ð8ð ð ð ð ð r   