§
    9²¸j°ª  ã                  ó€  — U d Z ddlmZ ddlZddlZddlZddlZddlZddlm	Z	 ddl
mZ ddlZddlmZ ddlmZ dd	lmZmZmZ d
dlmZmZ  ej        e¦  «        Z ed¬¦  «         G d„ d¦  «        ¦   «         Zej        d]d„¦   «         Zej        d^d„¦   «         Zej        j         d_d„¦   «         Z!ej        j         d`dad„¦   «         Z"da#de$d<   ej        j%        dbd„¦   «         Z&dcd„Z'ddd"„Z(ded$„Z)dfdgd(„Z*dhd-„Z+did4„Z,djd8„Z-dkd:„Z.dld@„Z/dmdJ„Z0 edKdL¬M¦  «        	 	 	 	 dndodS„¦   «         Z1dpdV„Z2dpdW„Z3dqdY„Z4	 dfdrd\„Z5dS )suS  DeepGEMM integration: fused grouped GEMM kernels from `kernels-community/deep-gemm`.

Provides:
- `deepgemm_bf16_experts_forward`: BF16 M-grouped experts forward.
- `deepgemm_fp8_fp4_linear`: end-to-end FP8/FP4 linear (output dtype follows the input).
- `deepgemm_fp8_fp4_experts_forward`: FP8 (or FP4 on SM100+) M-grouped experts forward.
- `deepgemm_fp8_fp4_megamoe_experts_forward`: FP8xFP4 Mega MoE forward (SM100+).

Requirements: CUDA, Hopper (SM90+), CUDA runtime â‰¥ 12.3, kernels-community/deep-gemm
â‰¥ 2.5 (Mega MoE symbols required). Mega MoE additionally needs SM100+ at call time.
é    )ÚannotationsN)ÚCallable)Ú	dataclassé   )Úlogging)Údeprecate_kwarg)Úis_kernels_availableÚmaybe_import_errorÚresolve_internal_importé   )Ú_MISSING_KERNELS_MESSAGEÚlazy_load_kernelT)Úfrozenc                  ó‚   — e Zd ZU dZded<   ded<   ded<   ded<   ded<   ded<   ded	<   ded
<   ded<   ded<   ded<   dS )ÚDeepGEMMz>Curated entry points exposed by `kernels-community/deep-gemm`.r   Úfp8_fp4_matmulÚgrouped_fp8_fp4_matmul_ntÚgrouped_fp8_fp4_matmul_nnÚgrouped_bf16_matmul_ntÚgrouped_bf16_matmul_nnÚper_token_cast_to_fp8Ú!transform_sf_into_required_layoutÚtransform_weights_for_mega_moeÚget_symm_buffer_for_mega_moeÚfp8_fp4_mega_moeÚintÚm_alignmentN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú__annotations__© ó    úr/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/integrations/deepgemm.pyr   r   6   sž   € € € € € € àHÐHàÐÐÑØ'Ð'Ð'Ñ'Ø'Ð'Ð'Ñ'Ø$Ð$Ð$Ñ$Ø$Ð$Ð$Ñ$Ø#Ð#Ð#Ñ#Ø/Ð/Ð/Ñ/Ø,Ð,Ð,Ñ,Ø*Ð*Ð*Ñ*ØÐÐÑð ÐÐÑÐÐr$   r   Úreturnú
str | Nonec                 óp  — t           j                             d¦  «        pt           j                             d¦  «        } | r| S t          j        d¦  «        }|r<t           j                             t           j                             |¦  «        ¦  «        S t           j                             d¦  «        rdS dS )u‚  Resolve the CUDA toolkit root the way DeepGEMM's JIT does:
    ``CUDA_HOME`` â†’ ``CUDA_PATH`` â†’ dir of ``which nvcc`` â†’ ``/usr/local/cuda`` (``None`` if none found).

    Mirrors DeepGEMM's own ``_find_cuda_home`` so we agree on the path it will actually use, rather than
    reusing ``torch.utils.cpp_extension.CUDA_HOME`` whose resolution inits a CUDA context (fork-unsafe).
    Ú	CUDA_HOMEÚ	CUDA_PATHÚnvccz/usr/local/cudaN)ÚosÚenvironÚgetÚshutilÚwhichÚpathÚdirnameÚisdir)Ú	cuda_homer+   s     r%   Ú_get_cuda_homer5   M   s“   € õ ”
—’˜{Ñ+Ô+ÐJ­r¬z¯~ª~¸kÑ/JÔ/J€IØð ØÐÝŒ<˜ÑÔ€DØð 6ÝŒw�Š�rœwŸš¨tÑ4Ô4Ñ5Ô5Ð5Ý	„w‡}‚}Ð&Ñ'Ô'ð !Ø Ð Øˆ4r$   útuple[int, int] | Nonec                 óÚ  — t          ¦   «         } | €dS t          j                             | d¦  «        }t          j                             |¦  «        rÕ	 t          |¦  «        5 }t          j        |¦  «        }ddd¦  «         n# 1 swxY w Y   |                     d|                     di ¦  «        ¦  «                             dd¦  «        }| 	                    d¦  «        dd…         \  }}t          |¦  «        t          |¦  «        fS # t          t          t          f$ r Y nw xY wt          j                             | d	¦  «        }t          j                             |¦  «        r­	 t          |¦  «        5 }t          j        d
|                     ¦   «         ¦  «        }ddd¦  «         n# 1 swxY w Y   |rDt          |                     d¦  «        ¦  «        t          |                     d¦  «        ¦  «        fS n# t          t          f$ r Y nw xY wt          j                             | dd¦  «        }	t          j                             |	¦  «        r˜	 t          |	¦  «        5 }t          j        d|                     ¦   «         ¦  «        }ddd¦  «         n# 1 swxY w Y   |r/t          |                     d¦  «        ¦  «        }
|
dz  |
dz  dz  fS n# t          t          f$ r Y nw xY wdS )a‹  Version of the CUDA toolkit nvcc will use, as ``(major, minor)``, read off disk without a
    subprocess from (in order) ``{CUDA_HOME}/version.json``, ``version.txt``, or the ``CUDA_VERSION``
    define in ``include/cuda.h``. ``None`` if unreadable. This is the compiler that builds the kernels,
    unlike ``torch.version.cuda`` (torch's bundled runtime, which never drives a JIT compile).
    Nzversion.jsonÚ	cuda_nvccÚcudaÚversionÚ ú.r   zversion.txtzCUDA Version (\d+)\.(\d+)r   Úincludezcuda.hz#define CUDA_VERSION (\d+)iè  é
   )r5   r,   r1   ÚjoinÚisfileÚopenÚjsonÚloadr.   Úsplitr   ÚOSErrorÚ
ValueErrorÚAttributeErrorÚreÚsearchÚreadÚgroup)r4   Úversion_jsonÚfÚ
componentsr:   ÚmajorÚminorÚversion_txtÚmatchÚcuda_hÚcuda_versions              r%   Ú_get_nvcc_versionrU   `   s�  € õ Ñ Ô €IØÐØˆtå”7—<’< 	¨>Ñ:Ô:€LÝ	„w‡~‚~�lÑ#Ô#ð ð	Ý�lÑ#Ô#ð * qÝ!œY q™\œ\�
ð*ð *ð *ñ *ô *ð *ð *ð *ð *ð *ð *øøøð *ð *ð *ð *à —n’n [°*·.².ÀÈÑ2LÔ2LÑMÔM×QÒQÐR[Ð]_Ñ`Ô`ˆGØ"Ÿ=š=¨Ñ-Ô-¨b¨q¨bÔ1‰LˆE�5Ý�u‘:”:�s 5™zœzÐ)Ð)øÝ�¥^Ð4ð 	ð 	ð 	ØˆDð	øøøõ ”'—,’,˜y¨-Ñ8Ô8€KÝ	„w‡~‚~�kÑ"Ô"ð ð	Ý�kÑ"Ô"ð J aÝœ	Ð">ÀÇÂÁÄÑIÔI�ðJð Jð Jñ Jô Jð Jð Jð Jð Jð Jð Jøøøð Jð Jð Jð Jàð @Ý˜5Ÿ;š; q™>œ>Ñ*Ô*­C°·²¸A±´Ñ,?Ô,?Ð?Ð?ð@øå�Ð$ð 	ð 	ð 	ØˆDð	øøøõ ŒW�\Š\˜) Y°Ñ9Ô9€FÝ	„w‡~‚~�fÑÔð ð	Ý�f‘”ð K Ýœ	Ð"?ÀÇÂÁÄÑJÔJ�ðKð Kð Kñ Kô Kð Kð Kð Kð Kð Kð Køøøð Kð Kð Kð Kàð IÝ" 5§;¢;¨q¡>¤>Ñ2Ô2�Ø# tÑ+¨l¸TÑ.AÀbÑ-HÐHÐHðIøõ �Ð$ð 	ð 	ð 	ØˆDð	øøøð ˆ4s¤   ÁD
 Á"BÁ7D
 ÂBÂD
 Â
BÂA>D
 Ä
D$Ä#D$Å'G< Å6(F*ÆG< Æ*F.Æ.G< Æ1F.Æ2AG< Ç<HÈHÉK É#(JÊK ÊJÊK ÊJÊ3K ËK(Ë'K(Úboolc                 óR   — t           j                             ¦   «         d         dk    S )u®  Whether the current CUDA device is Blackwell (SM100+). `@assume_constant_result` makes dynamo
    evaluate it once at trace time and inline the bool â€” keeping the untraceable
    `torch.cuda.get_device_capability()` pybind out of the graph â€” so the branches on it (SF layout, cast
    kwargs, psum layout, and the arch guards) stay compile-safe. Re-queries each eager call, so a faked
    capability in tests is honoured.
    r   r>   )Útorchr9   Úget_device_capabilityr#   r$   r%   Úis_sm100rZ   �   s"   € õ Œ:×+Ò+Ñ-Ô-¨aÔ0°BÒ6Ð6r$   FÚraise_errorc                óŠ  — t          ¦   «         st          dt          › �| ¬¦  «        S t          j                             ¦   «         st          d| ¬¦  «        S t          j                             ¦   «         \  }}|dvrt          d|› |› d�| ¬¦  «        S |dk    rdnd	}t          ¦   «         }|€$t          d|d         › d|d         › d�| ¬¦  «        S t          j	         
                    t          j	                             |dd¦  «        ¦  «        s't          d|› d|d         › d|d         › d�| ¬¦  «        S t          ¦   «         }|€'t          d|› d|d         › d|d         › d�| ¬¦  «        S ||k     rPt          d|› |› d|d         › d|d         › d|d         › d|d         › d|› d|d         › d|d         › d�| ¬¦  «        S dS )uì  Whether the DeepGEMM kernel can be loaded in this environment: `kernels` installed, a CUDA GPU on a
    supported arch (Hopper SM90 or Blackwell SM100), and a CUDA toolkit/nvcc new enough to JIT-compile it.
    A one-glance gate for callers â€” including external stacks â€” deciding whether to dispatch to DeepGEMM.
    With `raise_error=True` (used by the loader) it re-raises the specific `ImportError` explaining what's
    missing instead of returning `False`.

    `@assume_constant_result` makes dynamo evaluate this once at trace time and inline the bool (its probe
    is untraceable â€” an `lru_cache`'d import check + the `get_device_capability` pybind + nvcc lookup), so
    the check can sit in a compiled dispatch gate (e.g. `fp8_linear`) without breaking the graph.

    FP4 / Mega MoE additionally need Blackwell (SM100); the relevant forwards enforce that via
    `_assert_sm100_requirements` / `is_sm100()`, since this env check can't see which dtype path the
    caller will take.
    zDeepGEMM kernel unavailable: ©r[   z9DeepGEMM kernel requires CUDA, but CUDA is not available.)é	   r>   zJDeepGEMM requires Hopper (SM90) or Blackwell (SM100); current device is SMr<   r>   )é   r^   )r_   é   Nu(   DeepGEMM's JIT needs a CUDA toolkit â‰¥ r   r   z8, but none was found. Set `CUDA_HOME` to a CUDA toolkit.Úbinr+   z:DeepGEMM's JIT compiles with nvcc, but none was found in `u,   /bin`. Point `CUDA_HOME` at a full CUDA â‰¥ z& toolkit (not a runtime-only install).zDeepGEMM found nvcc in `u–   /bin` but could not read its CUDA version (no parseable `version.json`, `version.txt`, or `include/cuda.h`). Point `CUDA_HOME` at a complete CUDA â‰¥ z	 toolkit.zDeepGEMM on SMu    needs a CUDA â‰¥ z toolkit, but nvcc z in `u.   ` is too old. Point `CUDA_HOME` at a CUDA â‰¥ T)r	   r
   r   rX   r9   Úis_availablerY   r5   r,   r1   r@   r?   rU   )r[   rO   rP   Úmin_cudar4   Únvcc_versions         r%   Úis_deepgemm_loadablere   š   sÊ  € õ   Ñ!Ô!ð wÝ!Ð"\ÕBZÐ"\Ð"\ÐjuÐvÑvÔvÐvÝŒ:×"Ò"Ñ$Ô$ð xÝ!Ð"]ÐkvÐwÑwÔwÐwå”:×3Ò3Ñ5Ô5�L€Eˆ5ð �GÐÐÝ!ØhÐY^ÐhÐ`eÐhÐhÐhØ#ð
ñ 
ô 
ð 	
ð   2š+˜+ˆwˆw¨7€HÝÑ Ô €IØÐÝ!ð1°xÀ´{ð 1ð 1ÀXÈaÄ[ð 1ð 1ð 1à#ð
ñ 
ô 
ð 	
õ Œ7�>Š>�"œ'Ÿ,š, y°%¸Ñ@Ô@ÑAÔAð 
Ý!ðpÈð pð pØ.6°q¬kðpð pØ<DÀQ¼Kðpð pð pà#ð
ñ 
ô 
ð 	
õ %Ñ&Ô&€LØÐÝ!ðF yð Fð Fà!)¨!¤ðFð Fà/7¸¬{ðFð Fð Fð $ð	
ñ 
ô 
ð 	
ð �hÒÐÝ!ð=˜Uð = Eð =ð =¸XÀa¼[ð =ð =È8ÐTUÌ;ð =ð =Ø˜AŒð=ð =Ø!-¨a¤ð=ð =Ø7@ð=ð =à  œð=ð =à&.¨q¤kð=ð =ð =ð $ð	
ñ 
ô 
ð 	
ð ˆ4r$   zDeepGEMM | NoneÚ	_DEEPGEMMÚNonec                 óê  — t           �dS t          d¬¦  «         t          d¦  «        } | €t          d¦  «        ‚t	          | dd¦  «        }t	          | dd¦  «        }t	          | dd¦  «        }t	          | d	d¦  «        }t	          | d
d¦  «        }t          | d¬¦  «        }t	          | dd¦  «        }t	          | dd¦  «        }t	          | dd¦  «        }	t	          | dd¦  «        }
t	          | dd¦  «        }d„ d|fd|fd|fd	|fd
|fd|fd|fd|fd|	fd|
fd|ffD ¦   «         }|r-t          dd                     |¦  «        › dt          › �¦  «        ‚t          |||||||||	| |
¦   «         ¬¦  «        a dS )uý  Load DeepGEMM once into the `_DEEPGEMM` module global, raising `ImportError` if the env or any
    required symbol is missing. Under NO circumstances may this function return a value: it rides
    through `@allow_in_graph` as an opaque fx node, whose return must be proxyable â€” returning the
    bundle (e.g. from the warm-cache short-circuit) breaks torch.compile with
    `Unsupported: torch.* op returned non-Tensor`. Callers read `_DEEPGEMM` back from the global.

    `@allow_in_graph` makes `torch.compile` treat the untraceable cold path (hub download + dynamic
    import via `lazy_load_kernel`) as a single opaque node instead of tracing into it; it returns `None`
    (proxyable) and populates the global, which `load_deepgemm_kernel` then returns.
    NTr]   z	deep-gemmuc   Failed to load `kernels-community/deep-gemm` â€” check that a build matches the current torch/CUDA.Úfp8_fp4_gemm_ntÚ$m_grouped_fp8_fp4_gemm_nt_contiguousÚ$m_grouped_fp8_fp4_gemm_nn_contiguousÚ!m_grouped_bf16_gemm_nt_contiguousÚ!m_grouped_bf16_gemm_nn_contiguouszutils.per_token_cast_to_fp8)Úchained_pathr   r   r   Ú&get_mk_alignment_for_contiguous_layoutr   c                ó   — g | ]	\  }}|­|‘Œ
S ©Nr#   )Ú.0ÚnameÚattrs      r%   ú
<listcomp>z)_load_deepgemm_kernel.<locals>.<listcomp>	  s-   € ð ð ð áˆD�$ð ˆ<ð 	ð ˆ<ˆ<r$   z-DeepGEMM kernel is missing required symbols: z, z. )r   r   r   r   r   r   r   r   r   r   r   )	rf   re   r   ÚImportErrorÚgetattrr   r?   r   r   )Úkernelr   r   r   r   r   r   r   r   r   Úget_mk_alignmentr   Úmissings                r%   Ú_load_deepgemm_kernelr{   æ   s&  € õ ÐØˆå TÐ*Ñ*Ô*Ð*Ý˜kÑ*Ô*€FØ€~ÝØqñ
ô 
ð 	
õ ˜VÐ%6¸Ñ=Ô=€NÝ '¨Ð0VÐX\Ñ ]Ô ]ÐÝ '¨Ð0VÐX\Ñ ]Ô ]ÐÝ$ VÐ-PÐRVÑWÔWÐÝ$ VÐ-PÐRVÑWÔWÐÝ3°FÐIfÐgÑgÔgÐÝ(/°Ð8[Ð]aÑ(bÔ(bÐ%Ý%,¨VÐ5UÐW[Ñ%\Ô%\Ð"Ý#*¨6Ð3QÐSWÑ#XÔ#XÐ Ý˜vÐ'OÐQUÑVÔVÐÝ˜vÐ'9¸4Ñ@Ô@Ððð ð  Ð/Ø3Ð5NÐOØ3Ð5NÐOØ0Ð2HÐIØ0Ð2HÐIØ*Ð,AÐBØ0Ð2SÐTØ-Ð/MÐNØ+Ð-IÐJØ5Ð7GÐHØÐ!1Ð2ð
ðñ ô €Gð" ð 
ÝØl¸D¿IºIÀgÑ<NÔ<NÐlÐlÕRjÐlÐlñ
ô 
ð 	
õ Ø%Ø";Ø";Ø5Ø5Ø3Ø*KØ'EØ%AØ)Ø$Ð$Ñ&Ô&ðñ ô €I€I€Ir$   c                 ó,   — t          ¦   «          t          S rq   )r{   rf   r#   r$   r%   Úload_deepgemm_kernelr}   .  s   € ÝÑÔÐÝÐr$   Úweightútorch.TensorÚscalec                ó¶   — t          ¦   «         s&| j        t          j        k    rt	          d¦  «        ‚dS |j        t          j        k    rt	          d¦  «        ‚dS )um  Before-load guard for DeepGEMM's FP8/FP4 arch constraints on the given weight/scale dtypes:

      - FP4 (``int8``-packed) weights have no Hopper (SM90) kernel â€” they need Blackwell (SM100+).
      - Blackwell has no float32 scale-factor path: ``_coerce_sf_for_kernel`` would silently round a
        ``float32`` scale to UE8M0 and corrupt the output. UE8M0 scales load as ``float8_e8m0fnu`` (the
        loader normalizes even float32-container checkpoints like dsv4-flash-base), so a ``float32`` scale
        on SM100 means a genuine non-UE8M0 checkpoint.

    Uses `is_sm100()` (compile-safe via `assume_constant_result`), so the whole guard folds to a constant under
    ``torch.compile``: the valid case compiles away to nothing, while an unsupported combo fails loud
    rather than letting the hot path silently corrupt (unlike an ``is_compiling`` skip, which would miss a
    model compiled from cold with no eager warmup). Both raise ``NotImplementedError``, which
    ``fp8_linear`` treats as "DeepGEMM declined" and falls back to Triton (SM90 consuming float32 SFs
    directly is fine, so those cases are no-ops).
    z DeepGEMM's FP4 (int8-packed) path requires a Blackwell (SM100+) GPU; FP4 weights have no Hopper (SM90) kernel. Use an FP8 checkpoint, or run on a Blackwell GPU.Nu§  DeepGEMM has no float32 scale-factor path on Blackwell (SM100): these scales are plain float32 (quantization_config.scale_fmt='float'), and rounding them to UE8M0 would silently corrupt the output. Use a checkpoint quantized with scale_fmt='ue8m0', or a path that consumes float32 block scales directly â€” the FP8 linear falls back to Triton automatically; for experts use `model.set_experts_implementation('grouped_mm')`.)rZ   ÚdtyperX   Úint8ÚNotImplementedErrorÚfloat32)r~   r€   s     r%   Ú_assert_sm100_requirementsr†   6  so   € õ  ‰:Œ:ð àŒ<�5œ:Ò%Ð%Ý%ðZñô ð ð 	ˆð „{•e”mÒ#Ð#Ý!ð@ñ
ô 
ð 	
ð $Ð#r$   Úsfc                óª   — |                       t          j        ¦  «        }|dz                        d¦  «                              t          j        ¦  «        S )uÅ  Round each fp32 SF up to the nearest power of 2 (zero mantissa).

    Mirrors `deep_gemm.utils.math.ceil_to_ue8m0`. On SM100 the kernel's
    `pack_fp32_into_ue8m0` cleanly extracts the biased exponent only when the
    mantissa is already zero â€” its inner shifts (`>> 15`, `>> 7`, `<< 1`)
    otherwise leak mantissa bits into adjacent UE8M0 byte slots and silently
    corrupt the SF. SM90 consumes raw fp32 SFs without going through this path.
    iÿÿ i  €ÿ)ÚviewrX   Úint32Úbitwise_and_Úfloat)r‡   Úint_views     r%   Ú_ceil_to_ue8m0rŽ   Z  sA   € ð �wŠw•u”{Ñ#Ô#€HØ˜Ñ&×4Ò4Ð5EÑFÔF×KÒKÍEÌKÑXÔXÐXr$   rZ   Úexpected_mnú
int | Nonec                óè  — | j         t          j        k    r“|�H|                      d¦  «        |k     r/||                      d¦  «        z  }|                      |d¬¦  «        } |r2|                      ¦   «                              t          j        ¦  «        } n;|                      ¦   «         } n&| j         t          j	        k    r|rt          | ¦  «        } |                      ¦   «         dvr%t          d|                      ¦   «         › d�¦  «        ‚|s|                      ¦   «         S |                      d¦  «        }|                      d¦  «        }d|                      ¦   «         z  }| |z   |z  }|                      ¦   «         d	k    rd
|fn||z  d
|f}t          |                      ¦   «         ¦  «        |k    r| S t          j        | j        || j         | j        ¬¦  «        }	|	                     | ¦  «         |	S )u¢  Lay out `sf` as DeepGEMM's dispatch expects, per arch.

    On SM100 the int-SF path only *checks* the SF (`tma_stride_check`) and never
    transforms it, so we hand it a TMA-aligned MN-major layout (`stride(-2) == 1`,
    `stride(-1) == align(mn, 16/esize)`). On SM90 DeepGEMM transforms SFA itself
    (`get_mn_major_tma_aligned_tensor`) and only *checks* SFB against
    `sm90_sfb_check`, which rejects TMA padding (`stride(-1)` must equal `size(-2)`,
    not `align(mn, â€¦)`); a padded weight SF trips `layout.hpp` whenever `mn` isn't a
    multiple of `16/esize` (e.g. N=576 â†’ mn=5). So on SM90 we return the raw
    row-major SF and let DeepGEMM lay it out.

    Inputs come in three flavors:
      - `float8_e8m0fnu` on SM100: raw UE8M0 bytes â€” pack 4 K-bytes â†’ int32
        (last dim /4) for the kernel's `(INT, 1, gran_k)` path.
      - `float8_e8m0fnu` on SM90: SM90 dispatch only accepts FP32 SFs, so cast
        UE8M0 â†’ FP32 (exact upcast â€” UE8M0 is the biased-exponent half of a
        pow-of-2 FP32, so `.float()` rebuilds the original FP32 scale exactly).
      - `float32`: per-token / per-block SFs from `per_token_cast_to_fp8` or
        on-disk weights â€” round to UE8M0 on SM100 (see `_ceil_to_ue8m0`).
      - `int32`: already-packed UE8M0 â€” pass through.

    When `expected_mn` is set and the SF's M-dim is smaller (block-quantized
    UE8M0, e.g. DSv4-Flash compressor weights with `(N/128, K/128)` SFs), we
    repeat the SF on the M-axis to per-row before packing â€” the `(INT, 1, gran_k)`
    DeepGEMM kernel branch is the only UE8M0 path on SM100; for `gran_mn > 1`
    the kernel only handles FP32 SFs and would otherwise reject our INT SF here.
    Néþÿÿÿ©Údim)r   r`   z"DeepGEMM SF must be 2D or 3D, got ÚDéÿÿÿÿé   r   r   ©r‚   Údevice)r‚   rX   Úfloat8_e8m0fnuÚsizeÚrepeat_interleaveÚ
contiguousr‰   rŠ   rŒ   r…   rŽ   r”   rF   Úelement_sizeÚtupleÚstrideÚempty_stridedÚshaper™   Úcopy_)
r‡   rZ   r�   Úgran_mnÚmnÚkfÚalign_toÚ
aligned_mnÚtarget_stridesÚouts
             r%   Ú_coerce_sf_for_kernelr«   g  s¿  € ð8 
„x•5Ô'Ò'Ð'ØÐ" r§w¢w¨r¡{¤{°[Ò'@Ð'@Ø! R§W¢W¨R¡[¤[Ñ0ˆGØ×%Ò% g°2Ð%Ñ6Ô6ˆBØð 	Ø—’‘”×%Ò%¥e¤kÑ2Ô2ˆBˆBà—’‘”ˆBˆBØ	Œ•U”]Ò	"Ð	" xÐ	"Ý˜BÑÔˆà	‡v‚v�x„x�vÐÐÝÐI¸b¿fºf¹h¼hÐIÐIÐIÑJÔJÐJð ð Ø�}Š}‰ŒÐà	�Š�‰Œ€BØ	�Š�‰Œ€BØ�R—_’_Ñ&Ô&Ñ&€HØ�3˜(‘?Ð# hÑ.€JØ(*¯ª©¬°Aª¨�a˜�_�_¸BÀ¹OÈQÐPZÐ;[€NåˆR�YŠY‰[Œ[ÑÔ˜^Ò+Ð+Øˆ	Ý
Ô
˜bœh¨¸b¼hÈrÌyÐ
YÑ
YÔ
Y€CØ‡I‚Iˆb�M„M€MØ€Jr$   Úweight_scale_invÚ
block_sizeútuple | NoneÚdictc                óê   — | j         t          j        k    rddddœS |€t          d¦  «        ‚t	          |¦  «        }|dvrt          d|› d�¦  «        ‚|j         t          j        k    r|rdd	ddœS d
d	dœS )uC  Pick the `per_token_cast_to_fp8` kwargs from weight dtype + SF dtype + arch.

    Cases mirror the kernel's recipes:
      - FP4 weights (`int8`): gran_k=32 packed-UE8M0 SF. SM100+ only.
      - FP8 weights + UE8M0 SF on SM100: gran_k=128 packed-UE8M0 SF (DSv4).
      - FP8 weights + UE8M0 SF on SM90: gran_k=128 FP32 SF â€” the SM90 dispatch in
        `layout.hpp` only matches FP32 SFs, so we keep act SFs as FP32 (and float
        the weight SF in `_coerce_sf_for_kernel`; UE8M0 â†’ FP32 is an exact upcast).
      - FP8 weights + float SF: gran_k=128 float SF (DSv3).
    Té    ©Ú	use_ue8m0Úgran_kÚuse_packed_ue8m0Nz]DeepGEMM requires block-wise quantized FP8 weights, but the experts have no `block_size` set.))é€   r¶   )r   r¶   u?   DeepGEMM requires `block_size` âˆˆ {(128, 128), (1, 128)}, got r<   r¶   F)r³   r´   )r‚   rX   rƒ   rF   rŸ   rš   )r~   r¬   r­   rZ   s       r%   Ú_select_fp8_cast_kwargsr·   £  s¨   € ð „|•u”zÒ!Ð!Ø!¨RÀTÐJÐJÐJàÐÝØkñ
ô 
ð 	
õ �zÑ"Ô"€JØÐ/Ð/Ð/ÝÐjÐ]gÐjÐjÐjÑkÔkÐkØÔ¥Ô!5Ò5Ð5¸(Ð5Ø!¨SÀdÐKÐKÐKØ¨#Ð.Ð.Ð.r$   Úexpert_ids_sortedÚnum_expertsr   Ú	alignmentÚuse_psum_layoutú&tuple[torch.Tensor, torch.Tensor, int]c                ó  — | j         }|                      d¦  «        }|j        dk    r|                      ¦   «         n|                      ¦   «         }t          j        ||d|dz
  ¬¦  «                             ¦   «         }||z   dz
  |z  |z  }|t          ||¦  «        |dz
  z  z   }	||z
  }
t
          j	        j
                             |
                     d¦  «        d¦  «        }t          j        ||¬¦  «        ||          z   }|r(|                     d¦  «                             ¦   «         }nRt          j        |	fd|t
          j        ¬¦  «        }t          j        | |k     |                      ¦   «         d¦  «        ||<   |||	fS )	aŽ  Build the TMA-aligned grouped layout DeepGEMM expects.

    Returns `(sorted_to_padded, grouped_layout, total_padded_rows)`:
      - `grouped_layout` is per-row expert id (Hopper, with `-1` for padding /
        sentinels) or a cumsum of aligned per-expert counts (Blackwell).
      - EP sentinels (values == `num_experts`) are routed past the last expert
        block so DeepGEMM skips them.
    r   Úcpur   )ÚbinsÚminÚmax)r   r   ©r™   r–   ©r™   r‚   )r™   r›   ÚtyperŒ   r   rX   ÚhistcÚlongrÀ   ÚnnÚ
functionalÚpadÚcumsumÚarangeÚfullrŠ   Úwhere)r¸   r¹   rº   r»   r™   Ú
num_tokensÚhistc_inputÚtokens_per_expertÚaligned_tokens_per_expertÚtotal_padded_rowsÚpadding_per_expertÚcumulative_paddingÚsorted_to_paddedÚgrouped_layouts                 r%   Ú!_build_deepgemm_contiguous_layoutr×   Â  s“  € ð Ô%€FØ"×'Ò'¨Ñ*Ô*€Jð 06¬{¸eÒ/CÐ/CÐ#×)Ò)Ñ+Ô+Ð+ÐIZ×I^ÒI^ÑI`ÔI`€KÝœ K°kÀqÈkÐ\]ÉoÐ^Ñ^Ô^×cÒcÑeÔeÐØ"3°iÑ"?À!Ñ"CÈ	Ñ!QÐU^Ñ ^Ðà"¥S¨°[Ñ%AÔ%AÀYÐQRÁ]Ñ%SÑSÐð 3Ð5FÑFÐÝœÔ,×0Ò0Ð1C×1JÒ1JÈ1Ñ1MÔ1MÈvÑVÔVÐÝ”| J°vÐ>Ñ>Ô>ÐASÐTeÔAfÑfÐàð uØ2×9Ò9¸!Ñ<Ô<×@Ò@ÑBÔBˆˆåœÐ%6Ð$8¸"ÀVÕSXÔS^Ð_Ñ_Ô_ˆÝ+0¬;Ð7HÈ;Ò7VÐXi×XmÒXmÑXoÔXoÐqsÑ+tÔ+tˆÐ'Ñ(à˜^Ð->Ð>Ð>r$   ÚxrÕ   rÒ   c                ój   — t          j        |g| j        dd…         ¢R | j        | j        dœŽ}| ||<   |S )z;Pad a sorted tensor into the TMA-aligned contiguous layout.r   NrÃ   )rX   Úemptyr¢   r™   r‚   )rØ   rÕ   rÒ   Úpaddeds       r%   Ú_pad_for_deepgemmrÜ   æ  sD   € åŒ[Ð*ÐY¨Q¬W°Q°R°R¬[ÐYÐYÀÄÐQRÔQXÐYÐYÐY€FØ €FÐÑØ€Mr$   Úx_paddedc                ó   — | |         S rq   r#   )rÝ   rÕ   s     r%   Ú&_unpad_from_deepgemm_contiguous_layoutrß   í  s   € ØÐ$Ô%Ð%r$   Úhidden_statesÚtop_k_indexÚtop_k_weightsr   rŸ   c                óx  — |                      d¦  «        }|                     d¦  «        }|                     d¦  «        }t          j        |¦  «        \  }	}
| |
|z           }||
         }t	          |	|||¦  «        \  }}}|	|k                         d¦  «        }|	                     |dz
  ¬¦  «         |||	||
|||fS )zóSort tokens by expert id and build the M-grouped padded layout.

    Returns `(sorted_hidden_states_g, sample_weights_g, expert_ids_g,
              sentinel_mask, perm, sorted_to_padded, grouped_layout,
              total_padded_rows)`.
    r–   r   )rÁ   )r›   ÚreshaperX   Úsortr×   Ú	unsqueezeÚclamp_)rà   rá   râ   r¹   r   r»   Ú	num_top_kÚ
expert_idsÚsample_weightsÚexpert_ids_gÚpermÚsorted_hidden_states_gÚsample_weights_grÕ   rÖ   rÒ   Úsentinel_masks                    r%   Ú_dispatch_routed_inputrð   ô  sé   € ð × Ò  Ñ$Ô$€IØ×$Ò$ RÑ(Ô(€JØ"×*Ò*¨2Ñ.Ô.€Nõ œ JÑ/Ô/Ñ€L�$Ø*¨4°9Ñ+<Ô=ÐØ% dÔ+Ðõ
 ;\Ø�k ;°ñ;ô ;Ñ7Ð�nÐ&7ð " [Ò0×;Ò;¸BÑ?Ô?€MØ×Ò˜K¨!™OÐÑ,Ô,Ð,àØØØØØØØð	ð 	r$   Ú
out_paddedÚsorted_weightsrï   rì   rÎ   rè   Ú
hidden_dimÚ	out_dtypeútorch.dtypec	                óÀ  — t          | |¦  «        }	|	|                     |	j        ¦  «                             d¦  «        z  }
|
                     |d¦  «         t          j        |¦  «        }t          j        |                     d¦  «        |	j	        ¬¦  «        ||<   |
|          
                    |||¦  «                             d¬¦  «                             |¦  «        S )uR   Unpad â†’ weighted multiply â†’ mask sentinels â†’ restore order â†’ top-k reduce.r–   g        r   rÂ   r   r“   )rß   Útor‚   ræ   Úmasked_fill_rX   Ú
empty_likerË   r›   r™   r‰   Úsum)rñ   rò   rï   rì   rÕ   rÎ   rè   ró   rô   rª   ÚweightedÚinv_perms               r%   Ú_combine_routed_outputrý   &  sÅ   € õ 1°Ð=MÑ
NÔ
N€CØ�^×&Ò& s¤yÑ1Ô1×;Ò;¸BÑ?Ô?Ñ?€Hð ×Ò˜-¨Ñ-Ô-Ð-ÝÔ Ñ%Ô%€HÝ”\ $§)¢)¨A¡,¤,°s´zÐBÑBÔB€HˆT�Nà�HÔ×"Ò" :¨y¸*ÑEÔE×IÒIÈaÐIÑPÔP×SÒSÐT]Ñ^Ô^Ð^r$   Úoutput_dtypezv5.16)r:   ÚinputÚbiasútorch.Tensor | Noneútorch.dtype | NoneÚactivation_scalec                óª  — |�t          d¦  «        ‚| j        t          j        t          j        fvrt          d| j        › �¦  «        ‚t          ||¦  «         t          ¦   «         }t          |||t          ¦   «         ¦  «        }|  
                    d| j        d         ¦  «        }	 |j        |	fi |¤Ž\  }
}t          j        |
j        d         |j        d         | j        | j        ¬¦  «        }|                     d¦  «        rdd|d	         fnd}|                     |
t#          |t          ¦   «         |
                     d¦  «        ¬
¦  «        f|t#          |t          ¦   «         |                     d¦  «        ¬
¦  «        f||¬¦  «         | 
                    | j        dd…         |j        d         fz   ¦  «        }|�|                     |¦  «         |S )uó   End-to-end DeepGEMM linear: per-token activation quant + FP8/FP4 matmul.

    Static (per-tensor) activation quantization is rejected â€” DeepGEMM needs
    per-row SFs. Callers should route static activations through the Triton fallback.
    Nz@DeepGEMM linear does not support static activation quantization.z7DeepGEMM linear requires FP16 or BF16 activations, got r–   r   rÃ   rµ   r   r´   ©r�   )Úrecipe)r„   r‚   rX   Úbfloat16Úfloat16rF   r†   r}   r·   rZ   r‰   r¢   r   rÚ   r™   r.   r   r«   r›   Úadd_)rÿ   r~   r¬   r   r­   rþ   r  ÚdeepgemmÚcast_kwargsÚinput_2dÚ	qinput_2dÚscale_2dÚoutputÚ	sf_recipes                 r%   Údeepgemm_fp8_fp4_linearr  @  sÓ  € ð Ð#Ý!Ð"dÑeÔeÐeØ„{�5œ>­5¬=Ð9Ð9Ð9ÝÐ`ÐSXÔS^Ð`Ð`ÑaÔaÐaõ ˜vÐ'7Ñ8Ô8Ð8å#Ñ%Ô%€HÝ)¨&Ð2BÀJÕPXÑPZÔPZÑ[Ô[€Kà�zŠz˜"˜eœk¨"œoÑ.Ô.€HØ8˜(Ô8¸ÐQÐQÀ[ÐQÐQÑ€IˆxÝŒ[˜œ¨Ô+¨V¬\¸!¬_ÀUÄ\ÐY^ÔYdÐeÑeÔe€Fð 2=·²ÐASÑ1TÔ1TÐ^��A�{ 8Ô,Ð-Ð-ÐZ^€IØ×ÒØ	Õ)¨(µH±J´JÈIÏNÊNÐ[\ÑL]ÔL]Ð^Ñ^Ô^Ð_Ø	Õ&Ð'7½¹¼ÐQW×Q\ÒQ\Ð]^ÑQ_ÔQ_Ð`Ñ`Ô`ÐaØØð	 ñ ô ð ð �[Š[˜œ S b SÔ)¨V¬\¸!¬_Ð,>Ñ>Ñ?Ô?€FØÐØ�Š�DÑÔÐØ€Mr$   Úselfútorch.nn.Modulec                ó˜  — |j         t          j        k    rt          d|j         › �¦  «        ‚t	          ¦   «         }| j        r|j        n|j        }|j        }| 	                    d¦  «        }| 	                    d¦  «        }| 	                    d¦  «        }	t          |||| j        |j        t          ¦   «         ¦  «        \  }
}}}}}}}| j        r| j        n| j        }| j        }| j        r| j        r| j        n| j        nd }| j        r| j        nd }| j        r|j        d         n|j        d         }t/          |
||¦  «        }t          j        ||||j         ¬¦  «        } |||||t          ¦   «         ¬¦  «         | j        r|                     d|||         ¦  «         | j        r|                      |¦  «        n|                      |¦  «        }t          j        ||	||j         ¬¦  «        } |||||t          ¦   «         ¬¦  «         | j        r|                     d|||         ¦  «         t9          ||||||||	|j         ¦	  «	        S )Nú;DeepGEMM experts path requires bfloat16 hidden states, got r–   r   r   rÃ   )r»   )r‚   rX   r  rF   r}   Úis_transposedr   r   r™   r›   rð   r¹   r   rZ   Úhas_gateÚgate_up_projÚup_projÚ	down_projÚhas_biasÚgate_up_proj_biasÚup_proj_biasÚdown_proj_biasr¢   rÜ   rÚ   Ú
index_add_Ú_apply_gateÚact_fnrý   )r  rà   rá   râ   r
  Úgrouped_bf16_matmulr™   rè   rÎ   ró   Úsorted_hiddenrò   rë   rï   rì   rÕ   rÖ   rÒ   Ú	weight_upÚweight_downÚup_biasÚ	down_biasÚ
up_out_dimÚactÚproj_outrª   s                             r%   Údeepgemm_bf16_experts_forwardr+  m  s”  € ð Ô�eœnÒ,Ð,ÝÐlÐWdÔWjÐlÐlÑmÔmÐmå#Ñ%Ô%€Hà=AÔ=OÐt˜(Ô9Ð9ÐU]ÔUtÐàÔ!€FØ× Ò  Ñ$Ô$€IØ×#Ò# AÑ&Ô&€JØ×#Ò# BÑ'Ô'€Jõ 	Ø�{ M°4Ô3CÀXÔEYÕ[cÑ[eÔ[eñ	ô 	ñ	ØØØØØØØØð
 &*¤]ÐD�Ô!Ð!¸¼€IØ”.€KØRVÔR_Ði¨¬ÐMˆtÔ%Ð%¸DÔ<MÐ<MÐei€GØ'+¤}Ð>�Ô#Ð#¸$€Ið )-Ô(:ÐR�” Ô$Ð$À	ÄÐPQÔ@R€JÝ
˜MÐ+;Ð=NÑ
OÔ
O€CÝŒ{Ð,¨jÀÈ}ÔObÐcÑcÔc€HØÐ˜˜Y¨°.ÕRZÑR\ÔR\Ð]Ñ]Ô]Ð]Ø„}ð HØ×Ò˜AÐ/°¸Ô1FÑGÔGÐGà-1¬]ÐUˆt×Ò Ñ)Ô)Ð)ÀÇÂÈHÑ@UÔ@U€Hõ Œ+Ð'¨¸FÈ-ÔJ]Ð
^Ñ
^Ô
^€CØÐ˜ +¨s°NÕT\ÑT^ÔT^Ð_Ñ_Ô_Ð_Ø„}ð EØ�Š�qÐ*¨I°lÔ,CÑDÔDÐDå!ØØØØØØØØØÔñ
ô 
ð 
r$   c                ó�  — | j         rt          d¦  «        ‚t          | j        | j        ¦  «         t          ¦   «         }| j        dk    rt          d¦  «        ‚|j        t          j
        k    rt          d|j        › �¦  «        ‚| j        r|j        n|j        }|j        }|                     d¦  «        }|                     d¦  «        }|                     d¦  «        }	| j        r| j        n| j        }
| j        r| j        n| j        }| j        }| j        }t-          |
|| j        t1          ¦   «         ¦  «        }t3          |||| j        |j        t1          ¦   «         ¦  «        \  }}}}}}}}|                     d¦  «        rdd|d	         fnd } |j        |fi |¤Ž\  }}t=          |||¦  «        }t=          |||¦  «        }t          j        ||
j         d         |t          j
        ¬
¦  «        } ||tC          |t1          ¦   «         |¬¦  «        f|
tC          |t1          ¦   «         |
                     d¦  «        ¬¦  «        f|||t1          ¦   «         ¬¦  «         | j        r|  "                    |¦  «        n|  #                    |¦  «        } |j        |fi |¤Ž\  }}t          j        ||	|t          j
        ¬
¦  «        } ||tC          |t1          ¦   «         |¬¦  «        f|tC          |t1          ¦   «         |                     d¦  «        ¬¦  «        f|||t1          ¦   «         ¬¦  «         tI          ||||||||	|j        ¦	  «	        S )NzðDeepGEMM experts selected on a model spanning multiple CUDA devices in one process; its kernels are bound to a single CUDA context and corrupt across devices. Use `experts_implementation='grouped_mm'`, or run one device per process (TP/EP).ÚstaticzJDeepGEMM experts dispatch does not support static activation quantization.r  r–   r   rµ   r   r´   rÃ   r  r’   )r  r»   )%Ú_deepgemm_disabledÚRuntimeErrorr†   r  Údown_proj_scale_invr}   Úactivation_schemer„   r‚   rX   r  rF   r  r   r   r™   r›   r  r  r  Úgate_up_proj_scale_invÚup_proj_scale_invr·   r­   rZ   rð   r¹   r   r.   r   rÜ   rÚ   r¢   r«   r   r!  rý   )r  rà   rá   râ   r
  Úgrouped_fp8_fp4_matmulr™   rè   rÎ   ró   r$  Úweight_scale_upr%  Úweight_scale_downr  r#  rò   Ú_expert_ids_grï   rì   rÕ   rÖ   rÒ   r  Úact_fp8Ú
act_scalesr*  Úproj_fp8Úproj_scalesrª   s                                 r%   Ú deepgemm_fp8_fp4_experts_forwardr<  ®  s¨  € ð Ôð 
õ ð\ñ
ô 
ð 	
õ ˜tœ~¨tÔ/GÑHÔHÐHå#Ñ%Ô%€HàÔ Ò)Ð)Ý!Ð"nÑoÔoÐoØÔ�eœnÒ,Ð,ÝÐlÐWdÔWjÐlÐlÑmÔmÐmà.2Ô.@ÐhˆÔ*Ð*ÀhÔFhð ð Ô!€FØ× Ò  Ñ$Ô$€IØ×#Ò# AÑ&Ô&€JØ×#Ò# BÑ'Ô'€Jà%)¤]ÐD�Ô!Ð!¸¼€IØ59´]Ð^�dÔ1Ð1ÈÔH^€OØ”.€KØÔ0Ðå)¨)°_ÀdÄoÕW_ÑWaÔWaÑbÔb€Kõ 	Ø�{ M°4Ô3CÀXÔEYÕ[cÑ[eÔ[eñ	ô 	ñ	ØØØØØØØØð 2=·²ÐASÑ1TÔ1TÐ^��A�{ 8Ô,Ð-Ð-ÐZ^€Ið 9˜(Ô8¸ÐVÐVÈ+ÐVÐVÑ€GˆZÝ Ð)9Ð;LÑMÔM€GÝ" :Ð/?ÐARÑSÔS€JÝŒ{Ð,¨i¬o¸aÔ.@ÈÕW\ÔWeÐfÑfÔf€HØÐØ	Õ'¨
µH±J´JÐL]Ð^Ñ^Ô^Ð_Ø	Õ)¨/½8¹:¼:ÐS\×SaÒSaÐbdÑSeÔSeÐfÑfÔfÐgØØØÝ ™
œ
ðñ ô ð ð .2¬]ÐUˆt×Ò Ñ)Ô)Ð)ÀÇÂÈHÑ@UÔ@U€Hð ;˜HÔ:¸8ÐSÐSÀ{ÐSÐSÑ€HˆkÝ
Œ+Ð'¨¸FÍ%Ì.Ð
YÑ
YÔ
Y€CØÐØ	Õ(¨µh±j´jÐN_Ð`Ñ`Ô`ÐaØ	Õ+Ð,=½x¹z¼zÐWb×WgÒWgÐhjÑWkÔWkÐlÑlÔlÐmØØØÝ ™
œ
ðñ ô ð õ "ØØØØØØØØØÔñ
ô 
ð 
r$   Úmodulec                óä  — t          ¦   «         }| j        j        }| j        j        }| j        j                             t          j        ¦  «                             ¦   «         }| j	        j                             t          j        ¦  «                             ¦   «         }| j
        }| j        }| j        }|dz  dk    s	|dz  dk    rt          d|› d|› d�¦  «        ‚|                     |                     ¦   «         d|z  |d|¬¦  «        }	|                     |                     ¦   «         ||d|¬¦  «        }
|                     ||	f||
f¦  «        \  \  }}	\  }}
t          j                             |d	¬
¦  «        | _        t          j                             |	d	¬
¦  «        | _        t          j                             |d	¬
¦  «        | _	        t          j                             |
d	¬
¦  «        | _        dS )u!  One-shot pack + permute of an FP8Experts module's L1/L2 weights into the
    Mega MoE UTCCP layout. Called lazily on the first megamoe forward; idempotent
    via the caller's ``_megamoe_transformed`` flag.

    Steps:
      1. Cast UE8M0 SF â†’ FP32 and call ``transform_sf_into_required_layout`` â†’
         packed int32 in MN-major TMA-aligned layout.
      2. Run ``transform_weights_for_mega_moe``: interleaves gate/up on L1 and
         transposes both SFs for UTCCP.
      3. Overwrite the loader-side parameters in place; the interleave preserves
         the ``[E_local, 2*I, *]`` leading dims so downstream ``.size(...)`` reads
         stay valid.

    Unwraps any ``DTensor`` wrappers FSDP2/EP may have placed around the loader-
    side Parameters â€” the kernel takes raw pointers.
    r±   r   zwDeepGEMM Mega MoE requires `hidden_dim` and `intermediate_hidden` divisible by 32 (FP8 SF granularity); got hidden_dim=z, intermediate_hidden=r<   r   )r   r±   )r  Ú
num_groupsF)Úrequires_gradN)r}   r2  Údatar0  r  r‰   rX   rƒ   r�   r  Úintermediate_dimr¹   ró   rF   r   rŒ   r   rÇ   Ú	Parameter)r=  r
  Úgate_up_sf_rawÚdown_sf_rawÚ	gate_up_wÚdown_wÚintermediate_hiddenÚnum_local_expertsró   Ú
gate_up_sfÚdown_sfÚgate_upÚdowns                r%   Úsetup_megamoe_weightsrN    s  € õ" $Ñ%Ô%€HØÔ2Ô7€NØÔ,Ô1€KàÔ#Ô(×-Ò-­e¬jÑ9Ô9×DÒDÑFÔF€IØÔÔ"×'Ò'­¬
Ñ3Ô3×>Ò>Ñ@Ô@€Fà Ô1ÐØÔ*ÐØÔ"€Jà�B�˜!ÒÐÐ2°RÑ7¸1Ò<Ð<ÝðmØ4>ðmð mØViðmð mð mñ
ô 
ð 	
ð
 ×;Ò;Ø×ÒÑÔØ	ÐÑØØØ$ð <ñ ô €Jð ×8Ò8Ø×ÒÑÔØØØØ$ð 9ñ ô €Gð .6×-TÒ-TØ	�JÐØ	�Ðñ.ô .Ñ*Ñ€Wˆj™?˜D 'õ  œ(×,Ò,¨WÀEÐ,ÑJÔJ€FÔÝ$)¤H×$6Ò$6°zÐQVÐ$6Ñ$WÔ$W€FÔ!Ý”x×)Ò)¨$¸eÐ)ÑDÔD€FÔÝ!&¤×!3Ò!3°GÈ5Ð!3Ñ!QÔ!Q€FÔÐÐr$   Úprocess_groupú%torch.distributed.ProcessGroup | Nonec                ój  — t          | j        | j        ¦  «         | j        j        t          j        k    rt          d| j        j        › d�¦  «        ‚|€t          d¦  «        ‚t          ¦   «         }t          | dd¦  «        st          | ¦  «         d| _        |                     d¦  «        }|                     d	¦  «        }|                     d¦  «        }| j                             d	¦  «        }	| j                             d
¦  «        dz  }
|	|                     ¦   «         z  }t          | dd¦  «        �| j        j        |k     r |                     ||||||
¬¦  «        | _        |                     |ddd¬¦  «        \  }}| j        j        d|…                              |¦  «         | j        j        d|…                              |¦  «         | j        j        d|…                              |¦  «         | j        j        d|…                              |¦  «         t	          j        ||ft          j        |j        ¬¦  «        }|                     || j        | j        f| j        | j        f| j        t          t          | dd¦  «        dd¦  «        ¬¦  «         |                     |j        ¦  «        S )uï  FP8 acts Ã— FP4 weights Mega MoE forward (SM100+).

    Fuses EP dispatch + L1 + SwiGLU + L2 + EP combine into one kernel,
    overlapping NVLink with tensor-core compute. The kernel handles the full
    `(num_tokens, hidden) â†’ (num_tokens, hidden)` MoE forward including the
    weighted top-k reduction; the caller must NOT all-reduce the output.

    `process_group` is supplied automatically by `MoeTensorParalellExperts._prepare_input_fn`
    when the module is wrapped for TP â€” it's required for the symm-buffer rendezvous
    on first forward. `top_k_index` is GLOBAL expert ids (`-1` marks skipped slots).

    Caller-managed `self` attributes:
      - `gate_up_proj`, `gate_up_proj_scale_inv`: L1 weight + UE8M0 SF.
      - `down_proj`, `down_proj_scale_inv`: L2 weight + UE8M0 SF.
      Both pairs must be transformed together via
      `transform_weights_for_mega_moe((gate_up, gate_up_sf), (down, down_sf))`.
      - `config.swiglu_limit` (optional): SwiGLU clamp; absent â†’ unclamped.
    zJDeepGEMM Mega MoE requires FP4-packed expert weights (dtype=`int8`), got `z/`. Use the 'deepgemm' dispatch for FP8 experts.Nz©DeepGEMM Mega MoE requires a `process_group` for the EP group. The TP wrapping (MoeTensorParalellMegaMoeExperts) supplies it automatically; pass it explicitly otherwise.Ú_megamoe_transformedFTr–   r   r   r   Úsymm_buffer)ÚhiddenÚnum_topkr¹   Únum_max_tokens_per_rankrH  r±   r²   r˜   ÚconfigÚswiglu_limit)Úactivation_clamp)r†   r  r0  r‚   rX   rƒ   r„   rF   r}   rw   rN  rR  r›   rS  rV  r   r   rØ   r£   Úx_sfÚtopk_idxÚtopk_weightsrÚ   r  r™   r   r2  r  r÷   )r  rà   rá   râ   rO  r
  rè   rÎ   ró   rI  rH  Únum_global_expertsÚx_fp8rZ  Úys                  r%   Ú(deepgemm_fp8_fp4_megamoe_experts_forwardr`  E  sØ  € õ8 ˜tÔ0°$Ô2JÑKÔKÐKàÔÔ¥%¤*Ò,Ð,Ý!ðYØÔ!Ô'ðYð Yð Yñ
ô 
ð 	
ð
 ÐÝðiñ
ô 
ð 	
õ
 $Ñ%Ô%€Hõ �4Ð/°Ñ7Ô7ð )Ý˜dÑ#Ô#Ð#Ø$(ˆÔ!à× Ò  Ñ$Ô$€IØ×#Ò# AÑ&Ô&€JØ×#Ò# BÑ'Ô'€JØÔ)×.Ò.¨qÑ1Ô1ÐØÔ+×0Ò0°Ñ3Ô3°qÑ8ÐØ*¨]×-?Ò-?Ñ-AÔ-AÑAÐõ ˆt�] DÑ)Ô)Ð1°TÔ5EÔ5]Ð`jÒ5jÐ5jØ#×@Ò@ØØØØ*Ø$.Ø 3ð Añ 
ô 
ˆÔð ×0Ò0°È$ÐWYÐlpÐ0ÑqÔq�K€Eˆ4ØÔÔ�{˜
�{Ô#×)Ò)¨%Ñ0Ô0Ð0ØÔÔ˜+˜:˜+Ô&×,Ò,¨TÑ2Ô2Ð2ØÔÔ˜k˜z˜kÔ*×0Ò0°Ñ=Ô=Ð=ØÔÔ! + : +Ô.×4Ò4°]ÑCÔCÐCõ 	Œ�Z Ð,µE´NÈ=ÔK_Ð`Ñ`Ô`€AØ×ÒØ	Ø	Ô	˜DÔ7Ð8Ø	Œ˜Ô1Ð2ØÔÝ ¥¨¨x¸Ñ!>Ô!>ÀÐPTÑUÔUð ñ ô ð ð �4Š4�Ô#Ñ$Ô$Ð$r$   )r&   r'   )r&   r6   )r&   rV   )F)r[   rV   r&   rV   )r&   rg   )r&   r   )r~   r   r€   r   r&   rg   )r‡   r   r&   r   rq   )r‡   r   rZ   rV   r�   r�   r&   r   )
r~   r   r¬   r   r­   r®   rZ   rV   r&   r¯   )
r¸   r   r¹   r   rº   r   r»   rV   r&   r¼   )rØ   r   rÕ   r   rÒ   r   r&   r   )rÝ   r   rÕ   r   r&   r   )rà   r   rá   r   râ   r   r¹   r   r   r   r»   rV   r&   rŸ   )rñ   r   rò   r   rï   r   rì   r   rÕ   r   rÎ   r   rè   r   ró   r   rô   rõ   r&   r   )NNNN)rÿ   r   r~   r   r¬   r   r   r  r­   r6   rþ   r  r  r  r&   r   )
r  r  rà   r   rá   r   râ   r   r&   r   )r=  r  r&   rg   )r  r  rà   r   rá   r   râ   r   rO  rP  r&   r   )6r!   Ú
__future__r   Ú	functoolsrB   r,   rH   r/   Úcollections.abcr   Údataclassesr   rX   Úutilsr   Úutils.deprecationr   Úutils.import_utilsr	   r
   r   Úhub_kernelsr   r   Ú
get_loggerr   Úloggerr   Úcacher5   rU   Ú_dynamoÚassume_constant_resultrZ   re   rf   r"   Úallow_in_graphr{   r}   r†   rŽ   r«   r·   r×   rÜ   rß   rð   rý   r  r+  r<  rN  r`  r#   r$   r%   ú<module>ro     sº  ðð
ð 
ð 
ð #Ð "Ð "Ð "Ð "Ð "à Ð Ð Ð Ø €€€Ø 	€	€	€	Ø 	€	€	€	Ø €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !à €€€à Ð Ð Ð Ð Ð Ø /Ð /Ð /Ð /Ð /Ð /ðð ð ð ð ð ð ð ð ð ð
 DÐ CÐ CÐ CÐ CÐ CÐ CÐ Cð 
ˆÔ	˜HÑ	%Ô	%€ð
 €�$ÐÑÔðð ð ð ð ñ ô ñ Ôðð, „ðð ð ñ „ðð$ „ð+ð +ð +ñ „ð+ð\ „Ô%ð7ð 7ð 7ñ &Ô%ð7ð „Ô%ðBð Bð Bð Bñ &Ô%ðBðP "€	Ð !Ð !Ð !Ñ !ð „ÔðDð Dð Dñ ÔðDðNð ð ð ð!
ð !
ð !
ð !
ðH
Yð 
Yð 
Yð 
Yð9ð 9ð 9ð 9ð 9ðx/ð /ð /ð /ð>!?ð !?ð !?ð !?ðHð ð ð ð&ð &ð &ð &ð/ð /ð /ð /ðd_ð _ð _ð _ð4 €�¨Ð1Ñ1Ô1ð
 !%Ø)-Ø'+Ø,0ð)ð )ð )ð )ñ 2Ô1ð)ðX>ð >ð >ð >ðBZð Zð Zð Zðz7Rð 7Rð 7Rð 7Rð~ <@ðV%ð V%ð V%ð V%ð V%ð V%ð V%r$   