§
    9²¸j]Î  ã                  ób  — U d dl mZ d dlZd dlZd dlmZ d dlmZ d dlZd dl	m
Z
 d dl	mZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZ ddlmZmZmZmZmZ ddlmZm Z  ddl!m"Z"m#Z#  ej$        e%¦  «        Z&ej'        Z( ej)        e(¦  «        j*        Z+ ej)        e(¦  «        j,        Z-ej.        dYd„¦   «         Z/d„ Z0 ed¬¦  «         G d„ d¦  «        ¦   «         Z1da2de3d<   ej4        j5        dZd„¦   «         Z6d[d„Z7d\d „Z8	 	 	 	 d]d^d,„Z9	 	 	 d_d`d6„Z:	 	 	 	 dadbd9„Z; G d:„ d;e
j<        ¦  «        Z= G d<„ d=e
j>        ¦  «        Z? G d>„ d?e=¦  «        Z@dcdE„ZAdcdF„ZB G dG„ dHe
jC        ¦  «        ZD G dI„ dJe"¦  «        ZE eE¦   «         ZFdddM„ZGdedfdR„ZH	 dgdhdT„ZI G dU„ dVe¦  «        ZJ G dW„ dXe¦  «        ZKdS )ié    )ÚannotationsN)ÚCallable)Ú	dataclass)Ú
functionalé   )ÚACT2FN)ÚConversionOps)Úget_module_from_nameÚshould_convert_module)Úlogging)Úis_kernels_availableé   )Ú deepgemm_fp8_fp4_experts_forwardÚdeepgemm_fp8_fp4_linearÚ(deepgemm_fp8_fp4_megamoe_experts_forwardÚis_deepgemm_loadableÚis_sm100)Ú_MISSING_KERNELS_MESSAGEÚlazy_load_kernel)ÚExpertsInterfaceÚuse_experts_implementationÚreturnútorch.dtypec                 ó~   — t          t          d¦  «        st          dt          j        › d�¦  «        ‚t          j        S )uS  Return ``torch.float8_e8m0fnu`` or raise a clear error on torch without FP8 support.

    UE8M0 scales are always stored/consumed as this single dtype â€” the kernels (Triton
    finegrained + DeepGEMM) read it natively, and supporting the same scales in mixed
    container dtypes would be a mess â€” so fail loudly rather than fall back.Úfloat8_e8m0fnuzbscale_fmt='ue8m0' requires torch.float8_e8m0fnu, which is only available in PyTorch >= 2.7 (found z.). Upgrade torch to use UE8M0 FP8 checkpoints.)ÚhasattrÚtorchÚRuntimeErrorÚ__version__r   © ó    úy/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/integrations/finegrained_fp8.pyÚ_get_ue8m0_dtyper#   1   sR   € õ •5Ð*Ñ+Ô+ð 
ÝðgÝ%*Ô%6ðgð gð gñ
ô 
ð 	
õ ÔÐr!   c                óœ   — |D ]$}t          | |¦  «        rt          | |¦  «        c S Œ%t          t          | ¦  «        j        › d|› �¦  «        ‚)Nz has none of: )r   ÚgetattrÚAttributeErrorÚtypeÚ__name__)ÚobjÚnamesÚnames      r"   Ú_first_attrr,   @   sc   € Øð &ð &ˆÝ�3˜ÑÔð 	&Ý˜3 Ñ%Ô%Ð%Ð%Ð%ð	&å
�D ™IœIÔ.ÐEÐE¸eÐEÐEÑ
FÔ
FÐFr!   T)Úfrozenc                  ó2   — e Zd ZU dZded<   ded<   ded<   dS )ÚFineGrainedFP8zNEntry points exposed by the `kernels-community/finegrained-fp8` Triton kernel.r   ÚmatmulÚbatched_matmulÚgrouped_matmulN)r(   Ú
__module__Ú__qualname__Ú__doc__Ú__annotations__r    r!   r"   r/   r/   G   s<   € € € € € € àXÐXàÐÐÑØÐÐÑØÐÐÑÐÐr!   r/   zFineGrainedFP8 | NoneÚ_FINEGRAINED_FP8ÚNonec                 ó´  — t           �dS t          ¦   «         st          dt          › �¦  «        ‚t	          d¦  «        } | €t          d¦  «        ‚t          | dd¦  «        }t          | dd¦  «        }t          | dd¦  «        }d„ d|fd|fd|ffD ¦   «         }|r-t          d	d
                     |¦  «        › dt          › �¦  «        ‚t          |||¬¦  «        a dS )aç  
    Load the finegrained-fp8 Triton kernel once into the `_FINEGRAINED_FP8` module global.

    `@allow_in_graph` makes `torch.compile` treat the untraceable hub download + dynamic import as a
    single opaque node instead of tracing into it; it returns `None` (proxyable) and populates the
    global, which `load_finegrained_fp8_kernel` then returns.

    Under NO circumstances may this function return a value: an `@allow_in_graph` fx node's
    return must be proxyable, and returning the bundle (e.g. from the warm-cache
    short-circuit) breaks torch.compile with `Unsupported: torch.* op returned non-Tensor`.

    Raises `ImportError` if the `kernels` package is missing, or the kernel or required
    symbols cannot be found.
    Nz$finegrained-fp8 kernel unavailable: zfinegrained-fp8u‰   Failed to load the finegrained-fp8 kernel â€” check that `kernels-community/finegrained-fp8` has a build matching the current torch/CUDA.Ú	matmul_2dÚmatmul_batchedÚmatmul_groupedc                ó   — g | ]	\  }}|­|‘Œ
S ©Nr    )Ú.0r+   Úattrs      r"   ú
<listcomp>z0_load_finegrained_fp8_kernel.<locals>.<listcomp>x   s-   € ð ð ð áˆD�$ð
 ˆ<ð 	ð ˆ<ˆ<r!   z4finegrained-fp8 kernel is missing required symbols: ú, z. )r0   r1   r2   )r7   r   ÚImportErrorr   r   r%   Újoinr/   )Úkernelr0   r1   r2   Úmissings        r"   Ú_load_finegrained_fp8_kernelrG   V   s3  € õ" Ð#ØˆåÑ!Ô!ð ]ÝÐ[ÕAYÐ[Ð[Ñ\Ô\Ð\åÐ/Ñ0Ô0€FØ€~Ýð;ñ
ô 
ð 	
õ
 �V˜[¨$Ñ/Ô/€FÝ˜VÐ%5°tÑ<Ô<€NÝ˜VÐ%5°tÑ<Ô<€Nðð ð ˜&Ð!Ø˜~Ð.Ø˜~Ð.ð
ðñ ô €Gð ð 
ÝØsÀ4Ç9Â9ÈWÑCUÔCUÐsÐsÕYqÐsÐsñ
ô 
ð 	
õ &ØØ%Ø%ðñ ô ÐÐÐr!   c                 ó,   — t          ¦   «          t          S r>   )rG   r7   r    r!   r"   Úload_finegrained_fp8_kernelrI   �   s   € Ý Ñ"Ô"Ð"ÝÐr!   ÚaÚintÚbc                ó   — | |z   dz
  |z  S )zCeiling division.r   r    )rJ   rL   s     r"   Ú_cdivrN   ’   s   € à�‰E�A‰I˜!ÑÐr!   Únum_expertsÚproj_outÚproj_inÚweight_dtypeÚsf_dtypeÚweight_k_divÚ	sf_gran_nú
int | NoneÚ	sf_gran_kÚ
min_sf_outú!tuple[nn.Parameter, nn.Parameter]c	                ó|  — t          j        | |||z  |¬¦  «        }	t          j        |	|	                     ¦   «         ¬¦  «        }
t          |�t          ||¦  «        nd|¦  «        }|�t          ||¦  «        nd}t          j        | |||¬¦  «        }t          j        ||                     ¦   «         ¬¦  «        }|
|fS )u×  Allocate `(weight, weight_scale_inv)` parameters for one expert projection.

    `weight_k_div` halves the K dim for FP4-packed storage (2 e2m1 values per byte).
    `sf_gran_n` / `sf_gran_k` set per-block (None â†’ per-row/per-tensor) SF granularity.
    `min_sf_out` floors the SF tensor's output dim â€” used by the fused gate_up
    projection to keep room for both halves (pass `2`) even when `proj_out < sf_gran_n`
    would otherwise collapse the SF dim to 1.
    ©Údtype©Úrequires_gradNr   )r   ÚemptyÚnnÚ	ParameterÚis_floating_pointÚmaxrN   )rO   rP   rQ   rR   rS   rT   rU   rW   rX   Úweight_tÚweightÚsf_outÚsf_inÚsf_tÚsfs                  r"   Ú_alloc_expert_projrj   —   sÀ   € õ& Œ{˜;¨°'¸\Ñ2IÐQ]Ð^Ñ^Ô^€HÝŒ\˜(°(×2LÒ2LÑ2NÔ2NÐOÑOÔO€FÝ¨yÐ/D•�x Ñ+Ô+Ð+È!ÈZÑXÔX€FØ)2Ð)>�E�'˜9Ñ%Ô%Ð%ÀA€EÝŒ;�{ F¨E¸ÐBÑBÔB€DÝ	Œ�d¨$×*@Ò*@Ñ*BÔ*BÐ	CÑ	CÔ	C€BØ�2ˆ:Ðr!   Úinputútorch.Tensorre   Úweight_scale_invÚ
block_sizeúlist[int] | NoneÚbiasútorch.Tensor | NoneÚactivation_scalec                ó�   — t          ¦   «         }|                     | |||| j        |¬¦  «        }|�|                     |¦  «         |S )uu  Triton FP8/FP4 linear: fused act-quant + matmul, then optional bias add.

    ``activation_scale=None`` â†’ dynamic per-K-block scales (inline); set it for
    static per-tensor quant. ``weight_scale_inv`` accepts fp32 or UE8M0; the
    dispatcher routes FP4 (``int8``-packed) weights automatically. ``output_dtype``
    defaults to ``input.dtype`` when left ``None``.
    ©rr   )rI   r0   r\   Úadd_)rk   re   rm   rn   rp   rr   Úfinegrained_fp8Úoutputs           r"   Úfinegrained_fp8_linearrx   ³   s\   € õ 2Ñ3Ô3€OØ×#Ò#ØØØØØŒØ)ð $ñ ô €Fð ÐØ�Š�DÑÔÐØ€Mr!   Úallow_deepgemmÚboolc                ó   — |oŽt          ¦   «         o€|du o|t          ¦   «         o|j        t          j        k     oX|j        t          j        k    p|duo|d         |d         cxk    odk    nc o#t          j                             dd¦  «        dk    }|rX	 t          | |||||¬¦  «        S # t          t          t          f$ r(}t                               d	|› d
�¦  «         Y d}~nd}~ww xY wt          | |||||¦  «        S )u  End-to-end FP8/FP4 linear used by `FP8Linear` and the eager `FP8Experts` loop.

    Dispatch order â€” both backends handle FP8 and FP4 weights with fp32 or UE8M0 scales:
      1. DeepGEMM (`deepgemm_fp8_fp4_linear`) â€” 3-6Ã— faster on the shapes it supports.
         Preferred for FP4, UE8M0 SFs, and 128Ã—128 block FP8.
      2. Triton finegrained-fp8 fallback â€” used when DeepGEMM is unavailable, when the
         caller passes ``activation_scale`` (DeepGEMM is dynamic-only), or for any
         shape DeepGEMM declined.

    Args:
        input: (..., K) bf16/fp16 activations.
        weight: (N, K) `float8_e4m3fn` or (N, K // 2) `int8` (FP4-packed).
        weight_scale_inv: per-block weight scales â€” `float32` (V3-style) or `float8_e8m0fnu`
            (V4-style; reinterpreted as int32 at the DeepGEMM kernel boundary).
        block_size: [block_n, block_k] for FP8 block-wise quant, or None/[N, K] for per-tensor.
            Ignored for FP4 weights (the kernel infers SF granularity from the dtype).
        bias: optional bias added to the matmul output.
        activation_scale: pass a per-tensor scalar to use static activation quant; leave `None`
            for dynamic (per-token) quant.
        allow_deepgemm: set ``False`` to force the Triton fallback for this call. Used when the
            model spans multiple CUDA devices in one process â€” DeepGEMM's cached kernels are bound
            to a single CUDA context and produce garbage across devices (see the multi-device guard
            in ``quantizer_finegrained_fp8.py``).
    Nr   r   é€   Ú$TRANSFORMERS_DISABLE_DEEPGEMM_LINEARÚ0Ú1)rn   rr   rp   z=DeepGEMM declined this call, falling back to Triton. Reason: zW Set `TRANSFORMERS_DISABLE_DEEPGEMM_LINEAR=1` to skip DeepGEMM for FP8 linear entirely.)r   r   r\   r   Úfloat32Úint8ÚosÚenvironÚgetr   rC   ÚNotImplementedErrorÚ
ValueErrorÚloggerÚwarning_oncerx   )	rk   re   rm   rn   rp   rr   ry   Údeepgemm_preferredÚes	            r"   Ú
fp8_linearr‹   Ð   s„  € ðV 	ð 	OÝ Ñ"Ô"ð	Oà Ð$ð	Oõ ‘”ÐGÐ 0Ô 6½%¼-Ò GÐHð	Oð Œ\�UœZÒ'Ðm¨J¸dÐ,BÐ,lÀzÐRSÄ}ÐXbÐcdÔXeÐGlÐGlÒGlÐGlÐilÒGlÐGlÐGlÐGlð		Oõ
 ŒJ�NŠNÐAÀ3ÑGÔGÈ3ÒNð ð ð ð	Ý*ØØØ Ø%Ø!1Øðñ ô ð øõ Õ0µ*Ð=ð 	ð 	ð 	õ ×ÒðiÐPQð ið ið iñô ð ð ð ð ð ð øøøøð	øøøõ " %¨Ð1AÀ:ÈtÐUeÑfÔfÐfs   ÂB* Â*C)ÃC$Ã$C)c                  ó6   ‡ — e Zd ZdZ	 	 	 	 ddˆ fd„Zdd„Zˆ xZS )Ú	FP8LinearFNÚdynamicÚfloatÚin_featuresrK   Úout_featuresrn   útuple[int, int] | NoneÚactivation_schemeÚstrÚ	scale_fmtÚhas_biasrz   c                óî  •— t          ¦   «                              ||¦  «         || _        || _        || _        t
          j                             t          j        ||t          ¬¦  «        ¦  «        | _
        | j        €8t          j        t          j        dt
          j        ¬¦  «        ¦  «        | _        n—|dk    rt          ¦   «         nt
          j        }|| j        d         z   dz
  | j        d         z  }|| j        d         z   dz
  | j        d         z  }	t          j        t          j        ||	|¬¦  «        |j        ¬¦  «        | _        | j        dk    r8t          j        t          j        dt
          j        ¬¦  «        ¦  «        | _        n|                      dd ¦  «         | j        r2t          j        t          j        | j        ¦  «        ¦  «        | _        d S |                      d	d ¦  «         d S )
Nr[   ç      ð?Úue8m0r   r   r]   Ústaticrr   rp   )ÚsuperÚ__init__r–   rn   r“   r   r`   ra   r_   Ú
_FP8_DTYPEre   Útensorr€   rm   r#   rb   rr   Úregister_parameterr‘   rp   )Úselfr�   r‘   rn   r“   r•   r–   rS   Úscale_out_featuresÚscale_in_featuresÚ	__class__s             €r"   rœ   zFP8Linear.__init__#  s»  ø€ õ 	‰Œ×Ò˜ lÑ3Ô3Ð3à ˆŒØ$ˆŒØ!2ˆÔÝ”h×(Ò(­¬°\À;ÕV`Ð)aÑ)aÔ)aÑbÔbˆŒàŒ?Ð"å$&¤Lµ´¸cÍÌÐ1WÑ1WÔ1WÑ$XÔ$XˆDÔ!Ð!à-6¸'Ò-AÐ-AÕ'Ñ)Ô)Ð)ÅuÄ}ˆHØ".°´ÀÔ1CÑ"CÀaÑ"GÈDÌOÐ\]ÔL^Ñ!^ÐØ!,¨t¬¸qÔ/AÑ!AÀAÑ!EÈ$Ì/ÐZ[ÔJ\Ñ \ÐÝ$&¤LÝ”Ð.Ð0AÈÐRÑRÔRØ&Ô8ð%ñ %ô %ˆDÔ!ð
 Ô! XÒ-Ð-Ý$&¤Lµ´¸cÍÌÐ1WÑ1WÔ1WÑ$XÔ$XˆDÔ!Ð!à×#Ò#Ð$6¸Ñ=Ô=Ð=àŒ=ð 	2Ýœ¥U¤[°Ô1BÑ%CÔ%CÑDÔDˆDŒIˆIˆIà×#Ò# F¨DÑ1Ô1Ð1Ð1Ð1r!   rk   rl   r   c           	     óæ   — | j                              ¦   «         dk    r t          j        || j         | j        ¦  «        S t          || j         | j        | j        | j        | j        | j	         ¬¦  «        S )Nr   )rn   rr   rp   ry   )
re   Úelement_sizeÚFÚlinearrp   r‹   rm   rn   rr   Ú_deepgemm_disabled)r    rk   s     r"   ÚforwardzFP8Linear.forwardI  sq   € ØŒ;×#Ò#Ñ%Ô%¨Ò)Ð)Ý”8˜E 4¤;°´	Ñ:Ô:Ð:åØØŒKØÔ!Ø”Ø!Ô2Ø”Ø#Ô6Ð6ð
ñ 
ô 
ð 	
r!   ©NrŽ   r�   F)r�   rK   r‘   rK   rn   r’   r“   r”   r•   r”   r–   rz   ©rk   rl   r   rl   )r(   r3   r4   r¨   rœ   r©   Ú__classcell__©r£   s   @r"   r�   r�     sm   ø€ € € € € ð Ðð .2Ø!*Ø Øð$2ð $2ð $2ð $2ð $2ð $2ð $2ðL
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r!   r�   c                  ó4   ‡ — e Zd ZdZ	 ddˆ fd„Zdˆ fd„Zˆ xZS )ÚFP8Embeddingan  `nn.Embedding` whose table is stored in FP8 with a single per-tensor `weight_scale`.

    Qwen4-Exp's hashed n-gram table is ~48 GiB in FP8, too big to dequantize at load time, so the
    rescale runs on the rows a lookup gathers instead. Build it on the `meta` device: like
    `FP8Linear`, `__init__` lets `nn.Embedding` allocate a full-precision table first.
    NÚnum_embeddingsrK   Úembedding_dimÚpadding_idxrV   c                ó  •— t          ¦   «                              |||¬¦  «         t          j        t	          j        ||t          ¬¦  «        d¬¦  «        | _        t          j        t	          j        d¦  «        d¬¦  «        | _	        d S )N)r²   r[   Fr]   r   )
r›   rœ   r`   ra   r   r_   r�   re   ÚonesÚweight_scale)r    r°   r±   r²   r£   s       €r"   rœ   zFP8Embedding.__init__`  sr   ø€ õ 	‰Œ×Ò˜¨ÀKÐÑPÔPÐPÝ”l¥5¤;¨~¸}ÕT^Ð#_Ñ#_Ô#_ÐotÐuÑuÔuˆŒåœL­¬°A©¬ÀeÐLÑLÔLˆÔÐÐr!   rk   rl   r   c                ó”   •— t          ¦   «                              |¦  «        }|                     | j        j        ¦  «        | j        z  S r>   )r›   r©   Útorµ   r\   )r    rk   Úrowsr£   s      €r"   r©   zFP8Embedding.forwardk  s8   ø€ Ý‰wŒw�Š˜uÑ%Ô%ˆØ�wŠw�tÔ(Ô.Ñ/Ô/°$Ô2CÑCÐCr!   r>   )r°   rK   r±   rK   r²   rV   r«   ©r(   r3   r4   r5   rœ   r©   r¬   r­   s   @r"   r¯   r¯   X  s€   ø€ € € € € ðð ð #'ð		Mð 	Mð 	Mð 	Mð 	Mð 	Mð 	MðDð Dð Dð Dð Dð Dð Dð Dð Dð Dr!   r¯   c                  ó6   ‡ — e Zd ZdZ	 	 	 	 ddˆ fd„Zdd„Zˆ xZS )ÚFP8GroupedLinearuï  FP8 drop-in for block-diagonal grouped linears.

    The underlying nn.Linear stores a single `(n_groups * out_per_group, in_per_group)`
    weight; logically that's `n_groups` independent `(out_per_group, in_per_group)`
    sub-matrices, each consuming a disjoint slice of the input's last-but-one dim.
    Forward expects input of shape `(..., n_groups, in_per_group)` and returns
    `(..., n_groups, out_per_group)` â€” same contract as the vanilla bf16 grouped
    linear it replaces.

    NrŽ   r�   FÚin_features_per_grouprK   r‘   Ún_groupsrn   r’   r“   r”   r•   r–   rz   c                ód   •— t          ¦   «                              ||||||¬¦  «         || _        d S )N©r�   r‘   rn   r“   r•   r–   )r›   rœ   r½   )	r    r¼   r‘   r½   rn   r“   r•   r–   r£   s	           €r"   rœ   zFP8GroupedLinear.__init__|  sE   ø€ õ 	‰Œ×ÒØ-Ø%Ø!Ø/ØØð 	ñ 	
ô 	
ð 	
ð !ˆŒˆˆr!   Úxrl   r   c                óŽ  — |j         d d…         }|j         d         }| j                             ¦   «         dk    rà| j                             | j        d|¦  «                             dd¦  «        }|                     d| j        |¦  «                             dd¦  «        }t          j        ||¦  «                             dd¦  «        } |j        g |¢| j        ‘d‘R Ž }| j	        r3| 
                    | j                             | j        d¦  «        ¦  «         |S | j        }| j        }|                     | j        d|¦  «        }|                     dd¦  «                             d|¦  «        }|                     | j        |                     d¦  «        | j        z  |                     d¦  «        ¦  «        }|                     d¦  «        | j        z  }t          j        | j        f||j        t          j        ¬¦  «        }t          j        d| j        dz   |j        t          j        ¬¦  «        |z  }	t'          ¦   «         }
|
                     ||||	|| j        ¬¦  «        } |j        | j        g|¢d‘R Ž                      dd¦  «        }| j	        r3| 
                    | j                             | j        d¦  «        ¦  «         |S )Néþÿÿÿéÿÿÿÿr   r   r   )Údevicer\   ©ÚoffsetsÚtokens_per_expertrn   )Úshapere   r¥   Úviewr½   Ú	transposeÚreshaper   Úbmmr–   ru   rp   rm   ÚmovedimÚsizeÚfullrÄ   Úint32ÚarangerI   r2   rn   )r    rÀ   Úinput_shapeÚ
hidden_dimÚwÚyÚ	scale_invÚtokens_per_grouprÇ   rÆ   rv   s              r"   r©   zFP8GroupedLinear.forward�  s„  € Ø”g˜c˜r˜c”lˆØ”W˜R”[ˆ
àŒ;×#Ò#Ñ%Ô%¨Ò)Ð)Ø”× Ò  ¤°°JÑ?Ô?×IÒIÈ!ÈQÑOÔOˆAØ—	’	˜"˜dœm¨ZÑ8Ô8×BÒBÀ1ÀaÑHÔHˆAÝ”	˜!˜Q‘”×)Ò)¨!¨QÑ/Ô/ˆAØ�”	Ð:˜;Ð:¨¬Ð:°rÐ:Ð:Ð:ˆAØŒ}ð :Ø—’�t”y—~’~ d¤m°RÑ8Ô8Ñ9Ô9Ð9ØˆHàŒKˆØÔ)ˆ	à�FŠF�4”= " jÑ1Ô1ˆØ�IŠI�b˜!ÑÔ×$Ò$ R¨Ñ4Ô4ˆØ—N’N 4¤=°)·.².ÀÑ2CÔ2CÀtÄ}Ñ2TÐV_×VdÒVdÐefÑVgÔVgÑhÔhˆ	àŸ6š6 !™9œ9¨¬Ñ5ÐÝ!œJ¨¬Ð'7Ð9IÐRSÔRZÕbgÔbmÐnÑnÔnÐÝ”,˜q $¤-°!Ñ"3¸A¼HÍEÌKÐXÑXÔXÐ[kÑkˆå5Ñ7Ô7ˆØ×*Ò*ØØØØØ/Ø”ð +ñ 
ô 
ˆð ˆAŒI�d”mÐ6 kÐ6°2Ð6Ð6Ð6×>Ò>¸qÀ"ÑEÔEˆØŒ=ð 	6Ø�FŠF�4”9—>’> $¤-°Ñ4Ô4Ñ5Ô5Ð5Øˆr!   rª   )r¼   rK   r‘   rK   r½   rK   rn   r’   r“   r”   r•   r”   r–   rz   )rÀ   rl   r   rl   r¹   r­   s   @r"   r»   r»   p  sn   ø€ € € € € ð	ð 	ð  .2Ø!*Ø Øð!ð !ð !ð !ð !ð !ð !ð($ð $ð $ð $ð $ð $ð $ð $r!   r»   r    útorch.nn.ModuleÚhidden_statesÚtop_k_indexÚtop_k_weightsc                óô  — | j         dk    rt          d¦  «        ‚t          ¦   «         }|                     d¦  «        }|                     d¦  «        }|                     d¦  «        }|                     |d¬¦  «        }|                     d¦  «        }	|                     d¦  «        }
|
| j        k                         d¦  «        }| j        r| j	        n| j
        }| j        r| j        n| j        }| j        }| j        }|                     |||| j        |
¬¦  «        }| j        r|                      |¦  «        }n|                      |¦  «        }|                     |||| j        |
¬¦  «        }||	                     |j        ¦  «                             d¦  «        z  }|                     |d¦  «         |                     |||¦  «                             d¬¦  «        }|                     |j        ¦  «        S )	Nrš   z‘batched_mm experts dispatch does not support activation_scheme='static'. Use the default eager dispatch or switch to activation_scheme='dynamic'.rÃ   r   ©Údim)rn   Ú
expert_idsç        r   )r“   r…   rI   rÎ   Úrepeat_interleaverË   rO   Ú	unsqueezeÚhas_gateÚgate_up_projÚup_projÚgate_up_proj_scale_invÚup_proj_scale_invÚ	down_projÚdown_proj_scale_invr1   rn   Ú_apply_gateÚact_fnr·   r\   Úmasked_fill_rÉ   Úsum)r    rÙ   rÚ   rÛ   rv   Ú	num_top_kÚ
num_tokensrÓ   Úselected_hidden_statesÚsample_weightsrß   Úsentinel_maskÚ	weight_upÚweight_scale_upÚweight_downÚweight_scale_downrP   Úweighted_outÚfinal_hidden_statess                      r"   Úfp8_batched_mm_experts_forwardrù   ·  s  € ð Ô Ò)Ð)Ý!ðWñ
ô 
ð 	
õ
 2Ñ3Ô3€Oà× Ò  Ñ$Ô$€IØ×#Ò# AÑ&Ô&€JØ×#Ò# BÑ'Ô'€Jð +×<Ò<¸YÈAÐ<ÑNÔNÐØ"×*Ò*¨2Ñ.Ô.€NØ×$Ò$ RÑ(Ô(€Jð
   4Ô#3Ò3×>Ò>¸rÑBÔB€Mà%)¤]ÐD�Ô!Ð!¸¼€IØ59´]Ð^�dÔ1Ð1ÈÔH^€OØ”.€KØÔ0Ðð ×-Ò-ØØØØ”?Øð .ñ ô €Hð „}ð )à×#Ò# HÑ-Ô-ˆˆð —;’;˜xÑ(Ô(ˆð ×-Ò-ØØØØ”?Øð .ñ ô €Hð ˜n×/Ò/°´Ñ?Ô?×IÒIÈ"ÑMÔMÑM€Lð ×Ò˜m¨SÑ1Ô1Ð1ð '×+Ò+¨J¸	À:ÑNÔN×RÒRÐWXÐRÑYÔYÐà×!Ò! -Ô"5Ñ6Ô6Ð6r!   c                ó®  — | j         dk    rt          d¦  «        ‚t          ¦   «         }|j        }|                     d¦  «        }|                     d¦  «        }|                     d¦  «        }|                     d¦  «        }	|                     d¦  «        }
t          j        |
¦  «        \  }}|||z           }|	|         }|j        dk    r| 	                    ¦   «         n| 
                    ¦   «         }t          j        || j        d| j        dz
  ¬¦  «        }t          j        |dt          j        ¬¦  «        }|| j        k                         d¦  «        }| j        r| j        n| j        }| j        r| j        n| j        }| j        }| j        }|                     |||||| j        ¬	¦  «        }| j        r|                      |¦  «        }n|                      |¦  «        }|                     |||||| j        ¬	¦  «        }||                     |j        ¦  «                             d¦  «        z  }|                     |d
¦  «         t          j        |¦  «        }t          j        |                     d¦  «        |¬¦  «        ||<   ||         }|                      |||¦  «         !                    d¬¦  «        }|                     |j        ¦  «        S )Nrš   z‘grouped_mm experts dispatch does not support activation_scheme='static'. Use the default eager dispatch or switch to activation_scheme='dynamic'.rÃ   r   Úcpur   )ÚbinsÚminrc   )rÞ   r\   rÅ   rà   )rÄ   rÝ   )"r“   r…   rI   rÄ   rÎ   rË   r   Úsortr'   r�   rK   ÚhistcrO   ÚcumsumrÐ   râ   rã   rä   rå   ræ   rç   rè   ré   r2   rn   rê   rë   r·   r\   rì   Ú
empty_likerÑ   rÉ   rí   )r    rÙ   rÚ   rÛ   rv   rÄ   rî   rï   rÓ   rñ   rß   Úexpert_ids_gÚpermÚselected_hidden_states_gÚsample_weights_gÚhistc_inputrÇ   rÆ   rò   ró   rô   rõ   rö   rP   r÷   Úinv_permrø   s                              r"   Úfp8_grouped_mm_experts_forwardr    sò  € ð Ô Ò)Ð)Ý!ðWñ
ô 
ð 	
õ
 2Ñ3Ô3€OàÔ!€FØ× Ò  Ñ$Ô$€IØ×#Ò# AÑ&Ô&€JØ×#Ò# BÑ'Ô'€Jð #×*Ò*¨2Ñ.Ô.€NØ×$Ò$ RÑ(Ô(€Jõ œ JÑ/Ô/Ñ€L�$Ø,¨T°YÑ->Ô?ÐØ% dÔ+Ðð
 +1¬+¸Ò*>Ð*>�,×$Ò$Ñ&Ô&Ð&ÀL×DTÒDTÑDVÔDV€KÝœ K°dÔ6FÈAÐSWÔScÐfgÑSgÐhÑhÔhÐÝŒlÐ,°!½5¼;ÐGÑGÔG€Gð " TÔ%5Ò5×@Ò@ÀÑDÔD€Mà%)¤]ÐD�Ô!Ð!¸¼€IØ59´]Ð^�dÔ1Ð1ÈÔH^€OØ”.€KØÔ0Ðð ×-Ò-Ø ØØØØ+Ø”?ð .ñ ô €Hð „}ð )à×#Ò# HÑ-Ô-ˆˆð —;’;˜xÑ(Ô(ˆð ×-Ò-ØØØØØ+Ø”?ð .ñ ô €Hð Ð.×1Ò1°(´.ÑAÔA×KÒKÈBÑOÔOÑO€Lð ×Ò˜m¨SÑ1Ô1Ð1õ Ô Ñ%Ô%€HÝ”\ $§)¢)¨A¡,¤,°vÐ>Ñ>Ô>€HˆT�NØ Ô)€Lð '×+Ò+¨J¸	À:ÑNÔN×RÒRÐWXÐRÑYÔYÐà×!Ò! -Ô"5Ñ6Ô6Ð6r!   c                  óf   ‡ — e Zd ZU dZddddœiZded<   	 	 	 	 	 d#d$ˆ fd„Zd%d„Zd&d„Z	 d'd(d"„Z	ˆ xZ
S ))Ú
FP8ExpertsFÚdeepgemm_megamoeÚmegamoe_expertsÚmegamoe_router)Úmoe_tp_expertsÚ	ep_routerzdict[str, dict[str, str]]Ú_impl_tp_layer_overridesNrŽ   r�   Trn   r’   r“   r”   r•   r–   rz   rã   c                ó^  •— t          ¦   «                              ¦   «          |du s
J d¦   «         ‚|| _        || _        || _        || _        |j        | _        || _        t          |dd¦  «        | _
        t          |dd¦  «        | _        t          |dd ¦  «        | _        t          |dd ¦  «        | _        t          t          |d	d
¦  «                 | _        t          |dd ¦  «        | _        t          |dd¦  «        dk    }|dk    rt%          ¦   «         nt&          j        }|rt&          j        |ddddœ}	n t,          ||�|d         nd |�|d         nd dœ}	| j        rGt/          | j
        d| j        z  | j        fddi|	¤Ž\  | _        | _        |                      dd ¦  «         nAt/          | j
        | j        | j        fi |	¤Ž\  | _        | _        |                      dd ¦  «         t/          | j
        | j        | j        fi |	¤Ž\  | _        | _        |                      dd ¦  «         | j        dk    rzt?          j         t'          j!        | j
        t&          j        ¬¦  «        ¦  «        | _"        t?          j         t'          j!        | j
        t&          j        ¬¦  «        ¦  «        | _#        d S d S )NFzWFP8Experts does not support bias for now, please open an issue if you want this featureÚnum_local_expertsrO   Úmoe_intermediate_sizeÚintermediate_sizeÚswiglu_alphaÚswiglu_limitÚhidden_activationÚ
hidden_actÚexpert_dtypeÚfp8Úfp4r™   r   r   é    )rR   rS   rT   rU   rW   r   )rR   rS   rU   rW   rX   Úgate_up_proj_biasÚup_proj_biasÚdown_proj_biasrš   r[   )$r›   rœ   Úconfigr–   rã   rn   Úhidden_sizerÓ   r“   r,   rO   Úintermediate_dimr%   r  r  r   rë   Úlimitr#   r   r€   r�   r�   rj   rä   ræ   rŸ   rå   rç   rè   ré   r`   ra   r´   Úgate_up_proj_activation_scaleÚdown_proj_activation_scale)r    r   rn   r“   r•   r–   rã   Úis_fp4rS   Úalloc_kwargsr£   s             €r"   rœ   zFP8Experts.__init__s  sè  ø€ õ 	‰Œ×ÒÑÔÐà˜5Ð Ð Ð Øeñ !Ô Ð ð ˆŒØ ˆŒØ ˆŒØ$ˆŒØ Ô,ˆŒØ!2ˆÔÝ& vÐ/BÀMÑRÔRˆÔÝ +¨FÐ4KÐM`Ñ aÔ aˆÔÝ# F¨N¸DÑAÔAˆÔÝ# F¨N¸DÑAÔAˆÔÝ�[¨Ð1DÀlÑSÔSÔTˆŒÝ˜V ^°TÑ:Ô:ˆŒ
õ ˜ °Ñ7Ô7¸5Ò@ˆØ)2°gÒ)=Ð)=Õ#Ñ%Ô%Ð%Å5Ä=ˆØð 	å %¤
Ø$Ø !ØØðð ˆLˆLõ !+Ø$Ø.8Ð.D˜Z¨œ]˜]È$Ø.8Ð.D˜Z¨œ]˜]È$ð	ð ˆLð Œ=ð 		:Ý=OØÔ  ! dÔ&;Ñ";¸T¼_ð>ð >ØYZð>Ø^jð>ð >Ñ:ˆDÔ˜tÔ:ð ×#Ò#Ð$7¸Ñ>Ô>Ð>Ð>å3EØÔ  $Ô"7¸¼ð4ð 4ØLXð4ð 4Ñ0ˆDŒL˜$Ô0ð ×#Ò# N°DÑ9Ô9Ð9å3EØÔ˜dœo¨tÔ/Dð4
ð 4
ØHTð4
ð 4
Ñ0ˆŒ˜Ô0ð 	×ÒÐ 0°$Ñ7Ô7Ð7àÔ! XÒ-Ð-Ý13´½e¼jÈÔIYÕafÔanÐ>oÑ>oÔ>oÑ1pÔ1pˆDÔ.Ý.0¬l½5¼:ÀdÔFVÕ^cÔ^kÐ;lÑ;lÔ;lÑ.mÔ.mˆDÔ+Ð+Ð+ð .Ð-r!   Úgate_uprl   r   c                óÄ  — |                      dd¬¦  «        \  }}| j        �d|                     | j        ¬¦  «        }|                     | j         | j        ¬¦  «        }|t	          j        || j        z  ¦  «        z  }|dz   |z  S | j        �=|                     | j        ¬¦  «        }|                     | j         | j        ¬¦  «        }|                      |¦  «        |z  S )Nr   rÃ   rÝ   )rc   ©rý   rc   r˜   )Úchunkr  Úclampr  r   Úsigmoidr#  rë   )r    r(  ÚgateÚupÚglus        r"   rê   zFP8Experts._apply_gate¹  sÒ   € Ø—=’= ¨�=Ñ+Ô+‰ˆˆbØÔÐ(à—:’: $Ô"3�:Ñ4Ô4ˆDØ—’˜tÔ0Ð0°dÔ6G�ÑHÔHˆBØ�œ t¨dÔ.?Ñ'?Ñ@Ô@Ñ@ˆCØ˜‘H Ñ#Ð#ØŒZÐ#Ø—:’: $¤*�:Ñ-Ô-ˆDØ—’˜tœz˜k¨t¬z�Ñ:Ô:ˆBØ�{Š{˜4Ñ Ô  2Ñ%Ð%r!   rÙ   rÚ   rÛ   c                óê  — t          j        |t           j        ¬¦  «        }t          j        ¦   «         5  t           j        j                             || j        dz   ¬¦  «        }|                     ddd¦  «        }t          j	        | 
                    d¬¦  «        d¦  «                             d¬	¦  «                             d
¦  «        }d d d ¦  «         n# 1 swxY w Y   |D �]v}|| j        k    rŒt          j        ||         ¦  «        \  }}	||	         }
| j        dk    r| j        |         nd }|                      |
| j        r| j        |         n| j        |         | j        r| j        |         n| j        |         |¬¦  «        }| j        r|                      |¦  «        n|                      |¦  «        }| j        dk    r| j        |         nd }|                      || j        |         | j        |         |¬¦  «        }||	|d f         }||                     |j        ¦  «        z  }|                     d|	|                     |j        ¦  «        ¦  «         �Œx|                     |j        ¦  «        S )Nr[   r   )Únum_classesr   r   )rÃ   rÂ   rÝ   F)Úas_tuplerÃ   rš   rt   )r   Ú
zeros_liker€   Úno_gradr`   r   Úone_hotrO   ÚpermuteÚgreaterrí   ÚnonzerorÉ   Úwherer“   r$  r§   rã   rä   rå   ræ   rç   rê   rë   r%  rè   ré   r·   r\   Ú
index_add_)r    rÙ   rÚ   rÛ   rø   Úexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stateÚgate_up_act_scalerP   Údown_act_scaleÚrouting_weightsr÷   s                   r"   r©   zFP8Experts.forwardÆ  sË  € õ
 $Ô.¨}ÅEÄMÐRÑRÔRÐåŒ]‰_Œ_ð 	jð 	jÝœ(Ô-×5Ò5°kÈtÔO_ÐbcÑOcÐ5ÑdÔdˆKØ%×-Ò-¨a°°AÑ6Ô6ˆKÝœ {§¢¸8 Ñ'DÔ'DÀaÑHÔH×PÒPÐZ_ÐPÑ`Ô`×eÒeÐfhÑiÔiˆJð	jð 	jð 	jñ 	jô 	jð 	jð 	jð 	jð 	jð 	jð 	jøøøð 	jð 	jð 	jð 	jð
 %ð 	eñ 	eˆJØ˜TÔ-Ò-Ð-Øå#(¤;¨{¸:Ô/FÑ#GÔ#GÑ ˆI�yØ)¨)Ô4ˆMàBFÔBXÐ\dÒBdÐBd�Ô2°:Ô>Ð>Ðjnð ð —{’{ØØ15´Ð\�Ô! *Ô-Ð-ÀDÄLÐQ[ÔD\Ø;?¼=Ðp�Ô+¨JÔ7Ð7ÈdÔNdÐeoÔNpØ!2ð	 #ñ ô ˆHð 6:´]Ð]�t×'Ò'¨Ñ1Ô1Ð1ÈÏÊÐT\ÑH]ÔH]ˆHà?CÔ?UÐYaÒ?aÐ?a�Ô/°
Ô;Ð;Ðgkð ð —{’{ØØ”˜zÔ*ØÔ(¨Ô4Ø!/ð	 #ñ ô ˆHð ,¨I°yÀ$Ð,FÔGˆOØ# o×&8Ò&8¸¼Ñ&HÔ&HÑHˆLØ×*Ò*¨1¨i¸¿ºÐI\ÔIbÑ9cÔ9cÑdÔdÐdÑdØ"×%Ò% mÔ&9Ñ:Ô:Ð:s   ´BCÃCÃCrk   re   rm   rr   rq   c                óž   — |                      ¦   «         dk    rt          j        ||d ¦  «        S t          |||| j        || j         ¬¦  «        S )Nr   )rr   ry   )r¥   r¦   r§   r‹   rn   r¨   )r    rk   re   rm   rr   s        r"   r§   zFP8Experts.linearð  s`   € ð ×ÒÑ Ô  1Ò$Ð$Ý”8˜E 6¨4Ñ0Ô0Ð0åØØØØŒOØ-Ø#Ô6Ð6ð
ñ 
ô 
ð 	
r!   )NrŽ   r�   FT)
rn   r’   r“   r”   r•   r”   r–   rz   rã   rz   )r(  rl   r   rl   )rÙ   rl   rÚ   rl   rÛ   rl   r   rl   r>   )
rk   rl   re   rl   rm   rl   rr   rq   r   rl   )r(   r3   r4   r¨   r  r6   rœ   rê   r©   r§   r¬   r­   s   @r"   r
  r
  _  sä   ø€ € € € € € ð Ðð 	Ø/Ø)ð
ð 
ð;Ðð ð ð ñ ð .2Ø!*Ø ØØðDnð Dnð Dnð Dnð Dnð Dnð DnðL&ð &ð &ð &ð(;ð (;ð (;ð (;ð^ 15ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r!   r
  c                  ó    — e Zd ZdZeeeedœZdS )ÚFP8ExpertsInterfacez?Interface for registering custom FP8 experts forward functions.)Ú
batched_mmÚ
grouped_mmÚdeepgemmr  N)	r(   r3   r4   r5   rù   r  r   r   Ú_global_mappingr    r!   r"   rG  rG    s.   € € € € € ØIÐIð 5Ø4Ø4ØDð	ð €O€O€Or!   rG  Úmodelú	nn.Modulec                ó„  — d„ |                       ¦   «         D ¦   «         }t          ¦   «         }|D ]U}t          |                     ¦   «         d¦  «        }|�/|j        j        dk    r|                     |j        j        ¦  «         ŒVt          |¦  «        dk    rdS |D ]	}d|_	        Œ
t                               d¦  «         dS )a4  Internal, temporary helper (not public API): flag every FP8 module to skip DeepGEMM when the
    model spans >1 CUDA device in one process.

    DeepGEMM loads each kernel via `cuKernelGetFunction`, which binds the `CUfunction` handle to the
    CUDA context live at load time; driving that cached handle from another device launches it against
    the wrong context and produces garbage. (Build-time fix: compile DeepGEMM with
    `DG_JIT_USE_RUNTIME_API=1` for a context-free `cudaKernel_t` loader; until our wheel picks that up
    we avoid single-process multi-device.) Setting `_deepgemm_disabled` routes both the linear and
    experts paths through Triton/grouped_mm. A model that fits on one device keeps DeepGEMM even with
    other GPUs visible; TP/EP put one device per process, so this is a no-op there.
    c                óJ   — g | ] }t          |t          t          f¦  «        ¯|‘Œ!S r    )Ú
isinstancer�   r
  )r?   Úms     r"   rA   z5_disable_deepgemm_on_multi_device.<locals>.<listcomp>  s,   € ÐXÐXÐX˜µ¸AÅ	Í:Ð?VÑ1WÔ1WÐX�1ÐXÐXÐXr!   NÚcudar   TaB  This FP8 model spans multiple CUDA devices in one process; routing its FP8 linear and experts layers through Triton/grouped_mm instead of DeepGEMM (DeepGEMM's cached kernels are bound to a single CUDA context and corrupt across devices). Run tensor/expert parallel (one device per process) to use the faster DeepGEMM path.)ÚmodulesÚsetÚnextÚ
parametersrÄ   r'   ÚaddÚindexÚlenr¨   r‡   rˆ   )rL  Úfp8_modulesÚcuda_devicesrQ  Úparams        r"   Ú!_disable_deepgemm_on_multi_devicer]    s×   € ð YÐX˜eŸmšm™oœoÐXÑXÔX€KÝ‘5”5€LØð 1ð 1ˆÝ�Q—\’\‘^”^ TÑ*Ô*ˆØÐ ¤Ô!2°fÒ!<Ð!<Ø×Ò˜Uœ\Ô/Ñ0Ô0Ð0øÝ
ˆ<ÑÔ˜AÒÐØˆØð $ð $ˆØ#ˆÔÐÝ
×Òð	4ñô ð ð ð r!   Úpatternsú	list[str]Úmodules_to_not_convertúlist[str] | Nonec                ó¶  ‡— t          |                      ¦   «         ¦  «        D ]µ\  Š}t          |¦  «        t          j        u r•t          ˆfd„|D ¦   «         ¦  «        rzt          ‰|¦  «        sŒLt          j        d¦  «        5  t          |j
        |j        |j        ¦  «        }ddd¦  «         n# 1 swxY w Y   d|_        |                      ‰|¦  «         Œ¶| S )z½Swap every `nn.Embedding` whose name ends with one of `patterns` for `FP8Embedding`.

    Also runs under `dequantize=True`: a table worth quantizing is one we cannot afford to expand.
    c              3  óB   •K  — | ]}‰                      |¦  «        V — Œd S r>   )Úendswith)r?   Úpr+   s     €r"   ú	<genexpr>z-replace_with_fp8_embedding.<locals>.<genexpr>6  s/   øè è € Ð/SÐ/SÀQ°·²¸aÑ0@Ô0@Ð/SÐ/SÐ/SÐ/SÐ/SÐ/Sr!   ÚmetaNT)ÚlistÚnamed_modulesr'   r`   Ú	EmbeddingÚanyr   r   rÄ   r¯   r°   r±   r²   Ú_hf_quantized_needs_local_tpÚset_submodule)rL  r^  r`  ÚmoduleÚ
new_moduler+   s        @r"   Úreplace_with_fp8_embeddingrp  0  s,  ø€ õ
 ˜U×0Ò0Ñ2Ô2Ñ3Ô3ð 2ð 2‰ˆˆfÝ�‰<Œ<�2œ<Ð'Ð'­CÐ/SÐ/SÐ/SÐ/SÈ(Ð/SÑ/SÔ/SÑ,SÔ,SÐ'Ý(¨Ð/EÑFÔFð ØÝ”˜fÑ%Ô%ð kð kÝ)¨&Ô*?ÀÔAUÐW]ÔWiÑjÔj�
ðkð kð kñ kô kð kð kð kð kð kð køøøð kð kð kð kà6:ˆJÔ3Ø×Ò  jÑ1Ô1Ð1øØ€Ls   Â!B0Â0B4	Â7B4	Fc                óþ  — |j         r| S d}|                      ¦   «         D �]¿\  }}t          ||¦  «        sŒd}t          j        d¦  «        5  |                     d¦  «        rˆt          |dd¦  «        }t          |dd¦  «        }	t          |d| j                             ¦   «         ¦  «        }
t          t          t          |	|¬	¦  «        } ||
|j        |j        |j        |	|¬
¦  «        }n¼t          |¦  «        t           j        u r6t%          |j        |j        |j        |j        |j        |j        du¬¦  «        }nkt-          |t           j        ¦  «        rQdt          |¦  «        j        v r;t1          |j        |j        |j        |j        |j        |j        |j        du¬¦  «        }|�d|_        |                      ||¦  «         d}ddd¦  «         n# 1 swxY w Y   �ŒÁ|st8                               d¦  «         | S )a®  
    A helper function to replace all `torch.nn.Linear` modules by `FP8Linear` modules.

    Parameters:
        model (`torch.nn.Module`):
            Input model or `torch.nn.Module` as the function is run recursively.
        modules_to_not_convert (`list[`str`]`, *optional*, defaults to `None`):
            Names of the modules to not convert. In practice we keep the `lm_head` in full precision for numerical stability reasons.
        quantization_config (`FineGrainedFP8Config`):
            The quantization config object that contains the quantization parameters.
        pre_quantized (`bool`, defaults to `False`):
            Whether the model is pre-quantized or not
    FNrg  z.expertsrã   Tr–   r   )Úexperts_classÚexperts_interfacer–   rã   )r   rn   r“   r•   r–   rã   r¿   ÚGroupedLinear)r¼   r‘   r½   rn   r“   r•   r–   z�You are loading your model using fp8 but no linear modules were found in your model. Please double check your model architecture.)Ú
dequantizeri  r   r   rÄ   rd  r%   r   Úget_text_configr   r
  ÚALL_FP8_EXPERTS_FUNCTIONSÚweight_block_sizer“   r•   r'   r`   ÚLinearr�   r�   r‘   rp   rP  r(   r»   r½   rl  rm  r‡   Úwarning)rL  r`  Úquantization_configÚpre_quantizedÚhas_been_replacedÚmodule_namern  ro  rã   r–   r   Ú	new_classs               r"   Úreplace_with_fp8_linearr€  @  s€  € ð" Ô%ð ØˆàÐØ$×2Ò2Ñ4Ô4ð 6)ñ 6)Ñˆ�VÝ$ [Ð2HÑIÔIð 	Øàˆ
ÝŒ\˜&Ñ!Ô!ð 1	)ð 1	)Ø×#Ò# JÑ/Ô/ð +Ý" 6¨:°tÑ<Ô<�Ý" 6¨:°uÑ=Ô=�Ý  ¨°5´<×3OÒ3OÑ3QÔ3QÑRÔR�Ý6Ý",Ý&?Ø%Ø%ð	ñ ô �	ð '˜YØ!Ø2ÔDØ&9Ô&KØ1Ô;Ø%Ø%ðñ ô �
�
õ �f‘”¥¤Ð*Ð*å&Ø &Ô 2Ø!'Ô!4Ø2ÔDØ&9Ô&KØ1Ô;Ø#œ[°Ð4ðñ ô �
�
õ ˜F¥B¤IÑ.Ô.ð °?ÅdÈ6ÁlÄlÔF[Ð3[Ð3[õ .Ø*0Ô*<Ø!'Ô!4Ø#œ_Ø2ÔDØ&9Ô&KØ1Ô;Ø#œ[°Ð4ðñ ô �
ð Ð%à:>�
Ô7Ø×#Ò# K°Ñ<Ô<Ð<Ø$(Ð!ðc1	)ð 1	)ð 1	)ñ 1	)ô 1	)ð 1	)ð 1	)ð 1	)ð 1	)ð 1	)ð 1	)øøøð 1	)ð 1	)ð 1	)ð 1	)ùðf ð 
Ý�Šð<ñ	
ô 	
ð 	
ð €Ls   ÁE;GÇG	ÇG	c                  óH   — e Zd ZdZd„ Zdd„Zdd„Zdd„Zedd„¦   «         Z	dS )ÚFp8Quantizez^
    A quantization operation that creates two tensors, weight and scale out of a weight.
    c                ó   — || _         d S r>   ©Úhf_quantizer©r    r…  s     r"   rœ   zFp8Quantize.__init__š  ó   € Ø(ˆÔÐÐr!   Úvaluerl   r   útuple[int, int]c                ó(  — d }| j         j        �Zt          | j         j        t          ¦  «        r | j         j                             d¦  «        }nt          | j         j        dd ¦  «        }|€|j        d         |j        d         f}t          |¦  «        S )Nrx  rÂ   rÃ   )r…  r{  rP  Údictr„   r%   rÈ   Útuple)r    rˆ  rn   s      r"   Ú_resolve_block_sizezFp8Quantize._resolve_block_size�  s‹   € Øˆ
ØÔÔ0Ð<Ý˜$Ô+Ô?ÅÑFÔFð gØ!Ô.ÔB×FÒFÐGZÑ[Ô[�
�
å$ TÔ%6Ô%JÐL_ÐaeÑfÔf�
ØÐØœ+ bœ/¨5¬;°r¬?Ð;ˆJÝ�ZÑ Ô Ð r!   Úkeyr”   údict[str, torch.Tensor]c                ó°  — |j         dk     r||iS |                      |¦  «        \  }}|j        d         |j        d         }}||z  dk    s	||z  dk    r||iS |j        d d…         }||z  }||z  }	|j        }
|                     t          j        ¦  «        } |j        g |¢|‘|‘|	‘|‘R Ž }|                     ¦   «                              d¬¦  «        }t	          j	        |dk    |t	          j
        |¦  «        ¦  «        }t          |z  }t	          j	        |dk    |t	          j
        |¦  «        ¦  «        }d|z                       t          j        ¦  «        }| j        j        j        dk    r±t	          j        d	t	          j        t	          j        |                     t	          j        t          j        ¦  «        j        ¬
¦  «        ¦  «        ¦  «        ¦  «        }|                     t+          ¦   «         ¦  «        }d|                     t          j        ¦  «        z  }|                     d¦  «                             d¦  «        }||z  }t	          j        |t.          t          ¬¦  «                             t0          ¦  «        }|                     |
¦  «        }|                     d¦  «        r|                     dd¦  «        d         dz   n|dz   }||||iS )Nr   rÂ   rÃ   r   )éýÿÿÿrÃ   rÝ   r˜   r™   ç       @)rý   r‘  r*  ú.weightú.r   ú.weight_scale_invÚ
_scale_inv)Úndimr�  rÈ   r·   r   r€   rË   ÚabsÚamaxr:  Ú	ones_likeÚ_FP8_MAXr…  r{  r•   ÚpowÚceilÚlog2r,  ÚfinfoÚtinyr#   râ   Ú_FP8_MINr�   rd  Úrsplit)r    rŽ  rˆ  Úblock_mÚblock_nr¸   ÚcolsÚleading_shapeÚ
rows_tilesÚ
cols_tilesÚoriginal_shapeÚ
value_fp32ÚreshapedÚmax_absÚsafe_max_absÚscalesÚ
inv_scalesÚscales_broadcastÚscaledÚ	quantizedÚ	scale_keys                        r"   Ú_quantize_onezFp8Quantize._quantize_one¨  s™  € ð Œ:˜Š>ˆ>Ø˜�<ÐØ×3Ò3°EÑ:Ô:Ñˆ�Ø”[ ”_ e¤k°"¤oˆdˆØ�'‰>˜QÒÐ $¨¡.°AÒ"5Ð"5Ø˜�<Ðð œ C R CÔ(ˆØ˜W‘_ˆ
Ø˜W‘_ˆ
ØœˆØ—X’X�eœmÑ,Ô,ˆ
à%�:Ô%Ð_ }Ð_°jÐ_À'Ð_È:Ð_ÐW^Ð_Ð_Ð_ˆà—,’,‘.”.×%Ò%¨(Ð%Ñ3Ô3ˆÝ”{ 7¨Q¢;°½¼ÈÑ9QÔ9QÑRÔRˆå˜LÑ(ˆÝ”˜W qš[¨&µ%´/À&Ñ2IÔ2IÑJÔJˆØ˜F‘l×&Ò&¥u¤}Ñ5Ô5ˆ
ð ÔÔ0Ô:¸gÒEÐEÝœ 3­¬
µ5´:¸j×>NÒ>NÕSXÔS^Õ_dÔ_lÑSmÔSmÔSrÐ>NÑ>sÔ>sÑ3tÔ3tÑ(uÔ(uÑvÔvˆJØ#ŸšÕ'7Ñ'9Ô'9Ñ:Ô:ˆJØ˜:Ÿ=š=­¬Ñ7Ô7Ñ7ˆFà!×+Ò+¨BÑ/Ô/×9Ò9¸"Ñ=Ô=ÐØÐ,Ñ,ˆÝ”K ­H½(ÐCÑCÔC×FÒFÅzÑRÔRˆ	Ø×%Ò% nÑ5Ô5ˆ	ØCFÇ<Â<ÐPYÑCZÔCZÐr�C—J’J˜s AÑ&Ô& qÔ)Ð,?Ñ?Ð?Ð`cÐfrÑ`rˆ	Ø�Y 	¨:Ð6Ð6r!   Ú
input_dictc                óÎ   — i }|                      ¦   «         D ]M\  }}t          |t          ¦  «        r|d         n|}|                     |                      ||¦  «        ¦  «         ŒN|S )Nr   )ÚitemsrP  rh  Úupdater´  )r    rµ  ÚkwargsÚresultrŽ  rˆ  rž   s          r"   ÚconvertzFp8Quantize.convertÐ  sn   € ð +-ˆØ$×*Ò*Ñ,Ô,ð 	;ð 	;‰JˆC�Ý!+¨Eµ4Ñ!8Ô!8ÐC�U˜1”X�X¸eˆFØ�MŠM˜$×,Ò,¨S°&Ñ9Ô9Ñ:Ô:Ð:Ð:Øˆr!   r	   c                ó*   — t          | j        ¦  «        S r>   )ÚFp8Dequantizer…  ©r    s    r"   Ú
reverse_opzFp8Quantize.reverse_opÚ  s   € å˜TÔ.Ñ/Ô/Ð/r!   N)rˆ  rl   r   r‰  )rŽ  r”   rˆ  rl   r   r�  )rµ  rl   r   r�  ©r   r	   )
r(   r3   r4   r5   rœ   r�  r´  r»  Úpropertyr¿  r    r!   r"   r‚  r‚  •  sŠ   € € € € € ðð ð)ð )ð )ð	!ð 	!ð 	!ð 	!ð&7ð &7ð &7ð &7ðPð ð ð ð ð0ð 0ð 0ñ „Xð0ð 0ð 0r!   r‚  c                  óf   — e Zd ZdZd„ Zdd„ZdZdd
„Z	 ddd„Zdd„Z		 	 d d!d„Z
ed"d„¦   «         ZdS )#r½  u¬  Dequantize FP8 weights using their per-block ``weight_scale_inv``.

    Designed to run as the *first* op in any :class:`WeightConverter` chain when
    loading with ``dequantize=True`` â€” :meth:`update_weight_conversions` on the
    FP8 quantizer attaches it to each existing model-specific converter so that
    per-expert (weight, scale) pairs are folded into full-precision tensors before
    the chain's merge / concat ops collapse the per-expert structure.

    Pattern semantics
        Input ``input_dict`` carries one entry per source pattern; each value is a
        list of tensors (one per ``*`` match). For every weight pattern that has a
        sibling ``*.weight_scale_inv`` pattern in the dict, this op pairs them up by
        index, dequantizes per-pair, and emits the dequantized list under the
        original *weight* key. Scale entries are dropped from the output so the
        remaining ops only see weights.
    c                ó   — || _         d S r>   r„  r†  s     r"   rœ   zFp8Dequantize.__init__ñ  r‡  r!   Úweight_patternr”   r   c                óØ   — |                      d¦  «        }|r
|d d…         n|}|                      d¦  «        r|d t          d¦  «         …         dz   }n|dk    rd}n|dz   }|r|dz   n|S )Nú$rÃ   r“  r•  re   rm   r–  )rd  rY  )r    rÄ  ÚanchoredÚbaseÚscales        r"   Ú_scale_pattern_forz Fp8Dequantize._scale_pattern_forô  s�   € à!×*Ò*¨3Ñ/Ô/ˆØ&.ÐBˆ~˜c˜r˜cÔ"Ð"°NˆØ�=Š=˜Ñ#Ô#ð 	(ØÐ*�C 	™NœN˜?Ð*Ô+Ð.AÑAˆEˆEØ�XÒÐØ&ˆEˆEà˜<Ñ'ˆEØ&Ð1ˆu�s‰{ˆ{¨EÐ1r!   )rà   g      à?r˜   g      ø?r’  g      @g      @g      @g       €g      à¿g      ð¿g      ø¿g       Àg      Àg      Àg      ÀÚpackedrl   c                ó¸  — t          j        | j        t           j        |j        ¬¦  «        }|                     ¦   «                              t           j        ¦  «        }|dz                       ¦   «         }|dz	  dz                       ¦   «         }t          j	        ||         ||         gd¬¦  «        } |j
        g |j        dd…         ¢d|j        d         z  ‘R Ž S )uR   Two ``e2m1`` FP4 values per byte â†’ float32 tensor twice as wide on the last dim.)r\   rÄ   é   é   rÃ   rÝ   Nr   )r   rž   Ú_FP4_E2M1_LUTr€   rÄ   Ú
contiguousrÉ   Úuint8ÚlongÚstackrË   rÈ   )r    rË  ÚlutÚu8ÚlowÚhighÚunpackeds          r"   Ú_unpack_fp4zFp8Dequantize._unpack_fp4  sÂ   € åŒl˜4Ô-µU´]È6Ì=ÐYÑYÔYˆØ×ÒÑ Ô ×%Ò%¥e¤kÑ2Ô2ˆØ�C‰x�oŠoÑÔˆØ�q‘˜C‘×%Ò%Ñ'Ô'ˆÝ”;  C¤¨#¨d¬)Ð4¸"Ð=Ñ=Ô=ˆØˆxÔÐI ¤¨c¨r¨cÔ!2ÐI°A¸¼ÀRÔ8HÑ4HÐIÐIÐIÐIr!   Nr²  r®  Úoutput_dtypeútorch.dtype | Nonec                ó  — t          t          dd ¦  «        }|j        t          j        k    s|�!|j        |k    r|                      |¦  «        }n|                     t          j        ¦  «        }|j        dd …         \  }}	 |j        dd …         \  }}	n# t          $ r d\  }}	Y nw xY w||z  s||	z  rt          d|› d|› d|› d|	› d�	¦  «        ‚||z  }
||	z  }|€7|j        j
        r|                     ¦   «         dk    r|j        nt          j        }|j        t          j        k    r5|                     t          j        ¦  «        d	z
                       ¦   «         }n|                     t          j        ¦  «        }|j        }|                     d
||
|	|¦  «        }|                     d
||	¦  «                             d
¦  «                             d¦  «        }||z                       |¦  «                             |¦  «        S )NÚfloat4_e2m1fn_x2rÂ   )r   r   zWeight shape (rB   z) not divisible by scale grid (z).r   g     À_@rÃ   )r%   r   r\   r�   rÙ  r·   r€   rÈ   Ú	Exceptionr†   rb   r¥   Úbfloat16rÑ  Úexp2rË   râ   )r    r²  r®  rÚ  Ú	fp4_dtypeÚquantized_fp32r¸   r¥  Ú
scale_rowsÚ
scale_colsr£  r¤  Ús_fp32r©  ÚqÚss                   r"   Ú_dequantize_onezFp8Dequantize._dequantize_one  s   € õ
 �EÐ#5°tÑ<Ô<ˆ	ØŒ?�eœjÒ(Ð(¨YÐ-BÀyÄÐZcÒGcÐGcØ!×-Ò-¨iÑ8Ô8ˆNˆNà&Ÿ\š\­%¬-Ñ8Ô8ˆNØ#Ô)¨"¨#¨#Ô.‰
ˆˆdð	*Ø%+¤\°"°#°#Ô%6Ñ"ˆJ˜
˜
øÝð 	*ð 	*ð 	*à%)Ñ"ˆJ˜
˜
˜
ð	*øøøð �*Ñð 	  zÑ 1ð 	ÝØj ÐjÐj¨ÐjÐjÈjÐjÐjÐ\fÐjÐjÐjñô ð ð ˜*Ñ$ˆØ˜*Ñ$ˆð
 Ðà &¤Ô >ÐqÀ6×CVÒCVÑCXÔCXÐ\]ÒC]ÐC]�”�ÕchÔcqð ð Œ<�5œ;Ò&Ð&Ø—i’i¥¤Ñ.Ô.°Ñ6×<Ò<Ñ>Ô>ˆFˆFà—Y’Y�uœ}Ñ-Ô-ˆFØ'Ô-ˆØ×"Ò" 2 z°7¸JÈÑPÔPˆØ�NŠN˜2˜z¨:Ñ6Ô6×@Ò@ÀÑDÔD×NÒNÈqÑQÔQˆØ�A‘�zŠz˜,Ñ'Ô'×/Ò/°Ñ?Ô?Ð?s   ÂB ÂB&Â%B&rL  útorch.nn.Module | NoneÚfull_layer_nameú
str | Nonec                óx   — |�|€d S t          ||¦  «        \  }}t          ||d ¦  «        }t          |dd ¦  «        S )Nr\   )r
   r%   )r    rL  rê  rn  Útensor_namer\  s         r"   Ú_get_target_dtypezFp8Dequantize._get_target_dtype;  sH   € Øˆ=˜OÐ3Ø�4Ý2°5¸/ÑJÔJÑˆ�Ý˜ ¨TÑ2Ô2ˆÝ�u˜g tÑ,Ô,Ð,r!   rµ  ú,dict[str, list[torch.Tensor] | torch.Tensor]c                ó  ‡ ‡— ‰                       ||¦  «        Šd|v rv|�|nd}|d         }t          |t          ¦  «        r|d         n|}d|v rA|d         }t          |t          ¦  «        r|d         n|}|‰                      ||‰¬¦  «        iS ||iS i }|                     ¦   «         D ]Ü\  }	}
d|	v sd|	v rŒ‰                      |	¦  «        }||vr|
||	<   Œ-t          |
t          ¦  «        r|
n|
g}||         }t          |t          ¦  «        r|n|g}t          |¦  «        t          |¦  «        k    r3t          d|	› dt          |¦  «        › d	t          |¦  «        › d
�¦  «        ‚ˆˆ fd„t          ||¦  «        D ¦   «         ||	<   ŒÝ|S )Nzweight$re   r   rm   ©rÚ  rr   z/Fp8Dequantize: weight/scale count mismatch for z (z weights vs z	 scales).c                óF   •— g | ]\  }}‰                      ||‰¬ ¦  «        ‘ŒS )rñ  )rè  )r?   rÔ   rç  rÚ  r    s      €€r"   rA   z)Fp8Dequantize.convert.<locals>.<listcomp>m  s5   ø€ ÐrÐrÐrÑUYÐUVÐXY˜4×/Ò/°°1À<Ð/ÑPÔPÐrÐrÐrr!   )	rî  rP  rh  rè  r·  rÊ  rY  r†   Úzip)r    rµ  rê  rL  r¹  Ú
target_keyr²  r®  rº  rŽ  rˆ  r³  ÚweightsrÚ  s   `            @r"   r»  zFp8Dequantize.convertB  s  øø€ ð ×-Ò-¨e°_ÑEÔEˆð
 ˜
Ð"Ð"ð -<Ð,G˜˜ÈXˆJØ" 9Ô-ˆIÝ(2°9½dÑ(CÔ(CÐR˜	 !œ˜ÈˆIØ! ZÐ/Ð/Ø#Ð$6Ô7�Ý&0°½Ñ&>Ô&>ÐJ˜ œ˜ÀF�Ø" D×$8Ò$8¸ÀFÐYeÐ$8Ñ$fÔ$fÐgÐgØ 	Ð*Ð*ð @BˆØ$×*Ò*Ñ,Ô,ð 	sð 	s‰JˆC�Ø! SÐ(Ð(Ð,>À#Ð,EÐ,EØØ×/Ò/°Ñ4Ô4ˆIØ 
Ð*Ð*à#��s‘ØÝ)¨%µÑ6Ô6ÐC�e�e¸U¸GˆGØ 	Ô*ˆFÝ)¨&µ$Ñ7Ô7ÐE�V�V¸f¸XˆFÝ�7‰|Œ|�s 6™{œ{Ò*Ð*Ý ðIÀcð Ið IÝ˜G™œðIð IÝ25°f±+´+ðIð Ið Iñô ð ð sÐrÐrÐrÐrÕ]`ÐahÐjpÑ]qÔ]qÐrÑrÔrˆF�3‰KˆKØˆr!   r	   c                ó*   — t          | j        ¦  «        S r>   )r‚  r…  r¾  s    r"   r¿  zFp8Dequantize.reverse_opp  s   € õ
 ˜4Ô,Ñ-Ô-Ð-r!   )rÄ  r”   r   r”   )rË  rl   r   rl   r>   )r²  rl   r®  rl   rÚ  rÛ  r   rl   )rL  ré  rê  rë  r   rÛ  )NN)rµ  rï  rê  rë  rL  ré  r   rï  rÀ  )r(   r3   r4   r5   rœ   rÊ  rÏ  rÙ  rè  rî  r»  rÁ  r¿  r    r!   r"   r½  r½  ß  sÝ   € € € € € ðð ð")ð )ð )ð
2ð 
2ð 
2ð 
2ð m€MðJð Jð Jð Jð aeð+@ð +@ð +@ð +@ð +@ðZ-ð -ð -ð -ð '+Ø(,ð	,ð ,ð ,ð ,ð ,ð\ ð.ð .ð .ñ „Xð.ð .ð .r!   r½  )r   r   )r   r8   )r   r/   )rJ   rK   rL   rK   r   rK   )r   NNr   )rO   rK   rP   rK   rQ   rK   rR   r   rS   r   rT   rK   rU   rV   rW   rV   rX   rK   r   rY   )NNN)rk   rl   re   rl   rm   rl   rn   ro   rp   rq   rr   rq   r   rl   )NNNT)rk   rl   re   rl   rm   rl   rn   ro   rp   rq   rr   rq   ry   rz   r   rl   )
r    rØ   rÙ   rl   rÚ   rl   rÛ   rl   r   rl   )rL  rM  r   r8   r>   )r^  r_  r`  ra  )NNF)r`  ra  )LÚ
__future__r   Ú	functoolsr‚   Úcollections.abcr   Údataclassesr   r   Útorch.nnr`   r   r¦   Úactivationsr   Úcore_model_loadingr	   Úquantizers.quantizers_utilsr
   r   Úutilsr   Úutils.import_utilsr   rJ  r   r   r   r   r   Úhub_kernelsr   r   Úmoer   r   Ú
get_loggerr(   r‡   Úfloat8_e4m3fnr�   rŸ  rý   r¡  rc   r›  Úcacher#   r,   r/   r7   r6   Ú_dynamoÚallow_in_graphrG   rI   rN   rj   rx   r‹   ry  r�   rj  r¯   r»   rù   r  ÚModuler
  rG  rw  r]  rp  r€  r‚  r½  r    r!   r"   ú<module>r	     sô  ðð #Ð "Ð "Ð "Ð "Ð "Ð "à Ð Ð Ð Ø 	€	€	€	Ø $Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð Ø $Ð $Ð $Ð $Ð $Ð $à  Ð  Ð  Ð  Ð  Ð  Ø .Ð .Ð .Ð .Ð .Ð .Ø UÐ UÐ UÐ UÐ UÐ UÐ UÐ UØ Ð Ð Ð Ð Ð Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5ðð ð ð ð ð ð ð ð ð ð ð ð ð ð DÐ CÐ CÐ CÐ CÐ CÐ CÐ CØ =Ð =Ð =Ð =Ð =Ð =Ð =Ð =ð 
ˆÔ	˜HÑ	%Ô	%€ð Ô €
Øˆ5Œ;�zÑ"Ô"Ô&€Øˆ5Œ;�zÑ"Ô"Ô&€ð „ð ð  ð  ñ „ð ðGð Gð Gð €�$ÐÑÔðð ð ð ð ñ ô ñ Ôðð +/Ð Ð .Ð .Ð .Ñ .ð „Ôð3ð 3ð 3ñ Ôð3ðlð ð ð ð
ð ð ð ð Ø Ø Øðð ð ð ð ð@ $(Ø $Ø,0ðð ð ð ð ðB $(Ø $Ø,0ØðJgð Jgð Jgð Jgð JgðZ8
ð 8
ð 8
ð 8
ð 8
�”	ñ 8
ô 8
ð 8
ðvDð Dð Dð Dð D�2”<ñ Dô Dð Dð0Dð Dð Dð Dð D�yñ Dô Dð DðNG7ð G7ð G7ð G7ðT[7ð [7ð [7ð [7ð|b
ð b
ð b
ð b
ð b
�”ñ b
ô b
ð b
ðJð ð ð ð Ð*ñ ô ð ð 0Ð/Ñ1Ô1Ð ðð ð ð ð<ð ð ð ð ð" ejðRð Rð Rð Rð RðjG0ð G0ð G0ð G0ð G0�-ñ G0ô G0ð G0ðTV.ð V.ð V.ð V.ð V.�Mñ V.ô V.ð V.ð V.ð V.r!   