§
    “©ºjt   ã                   ó4  — d dl Z d dlmZ d dlmZmZ d dlmZmZm	Z	m
Z
mZ d dlZd dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZmZmZ d dlmZmZ erd dlmZ  ed¦  «        Z e j         e!¦  «        Z" G d„ de¦  «        Z#dej$        j%        ddfd„Z&dS )é    N)ÚMapping)ÚAbstractContextManagerÚ	ExitStack)ÚTYPE_CHECKINGÚAnyÚLiteralÚOptionalÚUnion)Úapply_to_collection)ÚRequirementCache)ÚTensor)Úoverride)Ú	Precision)Ú_ClassReplacementContextManagerÚ_convert_fp_tensorÚ_DtypeContextManager)Úrank_zero_infoÚrank_zero_warn©ÚDelayedScalingztransformer_engine>=0.11.0c                   óˆ  — e Zd ZU dZdZed         ed<   ddddœdej        de	e
eeef         d	f                  d
e	e         de	ej                 ddf
d„Zedej        j        dej        j        fd„¦   «         Zedefd„¦   «         Zedefd„¦   «         Zedefd„¦   «         Zededefd„¦   «         Zededefd„¦   «         ZdS )ÚTransformerEnginePrecisionaô  Plugin for training with fp8 precision via nvidia's
    `Transformer Engine <https://docs.nvidia.com/deeplearning/transformer-engine>`__.

    .. warning::  This is an :ref:`experimental <versioning:Experimental API>` feature.

    Args:
        weights_dtype: The weights dtype to use.
        recipe: Recipe for the DelayedScaling
            `configuration <https://docs.nvidia.com/deeplearning/transformer-engine/user-guide/api/common.html#transformer_engine.common.recipe.DelayedScaling>`__.
            In dict format or the dataclass format.
        replace_layers: Whether to replace ``Linear`` and ``LayerNorm`` layers automatically with their Transformer
            Engine alternatives. Note that they don't subclass the torch equivalents so checks like
            ``isinstance(l, torch.nn.Linear)`` will not pass.
        fallback_compute_dtype: The compute dtype to use for operations that don't support fp8 autocast. Defaults to the
            same as ``weights_dtype``.

    .. note::

        Support for FP8 in the linear layers with this plugin is currently limited to tensors
        with shapes where the dimensions are divisible by 8 and 16 respectively. You might want to add padding to your
        inputs to conform to this restriction.

    útransformer-engine)r   ztransformer-engine-float16Ú	precisionN)ÚrecipeÚreplace_layersÚfallback_compute_dtypeÚweights_dtyper   r   r   r   Úreturnc                óV  — t           s!t          t          t           ¦  «        ¦  «        ‚ddlm} |€ |¦   «         }nOt          |t          ¦  «        r:t          |¦  «        }d|v rddlm} t          ||d         ¦  «        |d<    |di |¤Ž}|| _
        || _        || _        |p|| _        d S )Nr   r   Ú
fp8_format)ÚFormat© )Ú_TRANSFORMER_ENGINE_AVAILABLEÚModuleNotFoundErrorÚstrÚ transformer_engine.common.reciper   Ú
isinstancer   Údictr"   Úgetattrr   r   r   r   )Úselfr   r   r   r   r   r"   s          ú…/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/lightning_fabric/plugins/precision/transformer_engine.pyÚ__init__z#TransformerEnginePrecision.__init__C   sÜ   € õ -ð 	JÝ%¥cÕ*GÑ&HÔ&HÑIÔIÐIØCÐCÐCÐCÐCÐCàˆ>Ø#�^Ñ%Ô%ˆFˆFÝ˜¥Ñ(Ô(ð 	.Ý˜&‘\”\ˆFØ˜vÐ%Ð%ØCÐCÐCÐCÐCÐCå'.¨v°v¸lÔ7KÑ'LÔ'L��|Ñ$Ø#�^Ð-Ð- fÐ-Ð-ˆFà*ˆÔØˆŒØ,ˆÔØ&<Ð&MÀˆÔ#Ð#Ð#ó    Úmodulec                 óô   — t          d„ |                     ¦   «         D ¦   «         ¦  «        r| j        du rt          d¦  «         n| j        dv rt	          |¦  «         |                     | j        ¬¦  «        }|S )Nc              3   ó(   K  — | ]}d |j         v V — ŒdS )ztransformer_engine.pytorchN)Ú
__module__)Ú.0Úms     r,   ú	<genexpr>z<TransformerEnginePrecision.convert_module.<locals>.<genexpr>a   s*   è è € ÐVÐVÀÐ+¨q¬|Ð;ÐVÐVÐVÐVÐVÐVr.   Tzz`TransformerEnginePrecision(replace_layers=True)` is set but the model already contains TransformerEngine layers. Skipping)NT)Údtype)ÚanyÚmodulesr   r   Ú_convert_layersÚtor   )r+   r/   s     r,   Úconvert_modulez)TransformerEnginePrecision.convert_module^   sŒ   € õ ÐVÐVÀVÇ^Â^ÑEUÔEUÐVÑVÔVÑVÔVð 	$ØÔ" dÐ*Ð*åð:ñô ð øð Ô  LÐ0Ð0Ý˜FÑ#Ô#Ð#Ø—’ Ô!3�Ñ4Ô4ˆØˆr.   c                 ó*   — t          | j        ¦  «        S )N)r   r   )r+   s    r,   Útensor_init_contextz.TransformerEnginePrecision.tensor_init_contextm   s   € å# DÔ$6Ñ7Ô7Ð7r.   c                 óð   — |                       ¦   «         }t          ¦   «         }| j        r7dd lm} t          |j        |j        dœ¦  «        }|                     |¦  «         |                     |¦  «         |S )Nr   )ztorch.nn.Linearztorch.nn.LayerNorm)	r=   r   r   Útransformer_engine.pytorchÚpytorchr   ÚLinearÚ	LayerNormÚenter_context)r+   Ú	dtype_ctxÚstackÚteÚcontext_managers        r,   Úmodule_init_contextz.TransformerEnginePrecision.module_init_contextq   s‹   € à×,Ò,Ñ.Ô.ˆ	Ý‘”ˆØÔð 	1Ø3Ð3Ð3Ð3Ð3Ð3å=Ø#%¤9Ø&(¤lð?ð ?ñ ô ˆOð ×Ò Ñ0Ô0Ð0Ø×Ò˜IÑ&Ô&Ð&Øˆr.   c                 óB  — t          | j        ¦  «        }t          j        d| j        ¬¦  «        }dd lm} |                     d| j        ¬¦  «        }t          ¦   «         }| 
                    |¦  «         | 
                    |¦  «         | 
                    |¦  «         |S )NÚcuda)Údevice_typer6   r   T)ÚenabledÚ
fp8_recipe)r   r   ÚtorchÚautocastr   r?   r@   Úfp8_autocastr   r   rC   )r+   rD   Úfallback_autocast_ctxrF   Úautocast_ctxrE   s         r,   Úforward_contextz*TransformerEnginePrecision.forward_context€   sž   € å(¨Ô);Ñ<Ô<ˆ	Ý %¤¸6ÈÔIdÐ eÑ eÔ eÐØ/Ð/Ð/Ð/Ð/Ð/à—’¨tÀÄ�ÑLÔLˆÝ‘”ˆØ×Ò˜IÑ&Ô&Ð&à×ÒÐ1Ñ2Ô2Ð2Ø×Ò˜LÑ)Ô)Ð)Øˆr.   Údatac                 óF   — t          |t          t          | j        ¬¦  «        S ©N)Úfunctionr6   Údst_type)r   r   r   r   ©r+   rT   s     r,   Úconvert_inputz(TransformerEnginePrecision.convert_inputŽ   s   € å" 4Õ2DÍFÐ]aÔ]oÐpÑpÔpÐpr.   c                 ó^   — t          |t          t          t          j        ¦   «         ¬¦  «        S rV   )r   r   r   rN   Úget_default_dtyperY   s     r,   Úconvert_outputz)TransformerEnginePrecision.convert_output’   s$   € å" 4Õ2DÍFÕ]bÔ]tÑ]vÔ]vÐwÑwÔwÐwr.   )Ú__name__r2   Ú__qualname__Ú__doc__r   r   Ú__annotations__rN   r6   r	   r
   r   r&   r   Úboolr-   r   ÚnnÚModuler;   r   r=   rH   rS   rZ   r]   r#   r.   r,   r   r   (   sà  € € € € € € ðð ð0 Nb€IˆwÐIÔJÐaÐaÑað HLØ)-Ø8<ðNð Nð Nð ”{ðNð ˜˜w s¨C xÔ0Ð2BÐBÔCÔDð	Nð
 ! œðNð !)¨¬Ô 5ðNð 
ðNð Nð Nð Nð6 ð U¤X¤_ð ¸¼¼ð ð ð ñ „Xðð ð8Ð%;ð 8ð 8ð 8ñ „Xð8ð ðÐ%;ð ð ð ñ „Xðð ðÐ!7ð ð ð ñ „Xðð ðq #ð q¨#ð qð qð qñ „Xðqð ðx 3ð x¨3ð xð xð xñ „Xðxð xð xr.   r   r/   r   c                 ó   — dd l m} |                      ¦   «         D �]ñ\  }}t          |t          j        j        ¦  «        rã|j        dz  dk    s|j        dz  dk    rt          d|›d�¦  «         ŒU|j
        d u}|                     |j        |j        |¬¦  «        }|j        j                             ¦   «         |j        _        |r(|j
        j                             ¦   «         |j
        _        t                               d|›d�¦  «         |                      ||¦  «         �Œt          |t          j        j        ¦  «        r»|                     |j        d         |j        ¬	¦  «        }|j        j                             ¦   «         |j        _        |j
        �/|j
        �(|j
        j                             ¦   «         |j
        _        t                               d|›d�¦  «         |                      ||¦  «         �Œât)          |¦  «         �Œód S )
Nr   é   é   z©Support for FP8 in the linear layers with this plugin is currently limited to tensors with shapes where the dimensions are divisible by 8 and 16 respectively. The layer zJ does not fit this criteria. You might want to add padding to your inputs.)ÚbiaszReplacing layer z# with Transformer Engine equivalent)Úeps)r?   r@   Únamed_childrenr(   rN   rc   rA   Úin_featuresÚout_featuresr   rh   ÚweightrT   ÚcloneÚlogÚdebugÚ__setattr__rB   Únormalized_shaperi   r9   )r/   rF   ÚnameÚchildÚhas_biasÚreplacements         r,   r9   r9   —   s  € Ø+Ð+Ð+Ð+Ð+Ð+à×,Ò,Ñ.Ô.ð #ñ #‰ˆˆeÝ�e�UœXœ_Ñ-Ô-ð 	#ØÔ  1Ñ$¨Ò)Ð)¨UÔ-?À"Ñ-DÈÒ-IÐ-Iåðuà"&ðuð uð uñô ð ð
 Ø”z¨Ð-ˆHØŸ)š) EÔ$5°uÔ7IÐPX˜)ÑYÔYˆKØ&+¤lÔ&7×&=Ò&=Ñ&?Ô&?ˆKÔÔ#Øð @Ø(-¬
¬×(=Ò(=Ñ(?Ô(?�Ô Ô%Ý�IŠIÐT¨ÐTÐTÐTÑUÔUÐUØ×Ò˜t [Ñ1Ô1Ð1Ñ1Ý˜�uœxÔ1Ñ2Ô2ð 	#ØŸ,š, uÔ'=¸aÔ'@ÀeÄi˜,ÑPÔPˆKØ&+¤lÔ&7×&=Ò&=Ñ&?Ô&?ˆKÔÔ#àŒzÐ%¨+Ô*:Ð*FØ(-¬
¬×(=Ò(=Ñ(?Ô(?�Ô Ô%Ý�IŠIÐT¨ÐTÐTÐTÑUÔUÐUØ×Ò˜t [Ñ1Ô1Ð1Ñ1õ ˜EÑ"Ô"Ð"Ñ"ð9#ð #r.   )'ÚloggingÚcollections.abcr   Ú
contextlibr   r   Útypingr   r   r   r	   r
   rN   Úlightning_utilitiesr   Ú lightning_utilities.core.importsr   r   Útyping_extensionsr   Ú,lightning_fabric.plugins.precision.precisionr   Ú(lightning_fabric.plugins.precision.utilsr   r   r   Ú$lightning_fabric.utilities.rank_zeror   r   r'   r   r$   Ú	getLoggerr^   ro   r   rc   rd   r9   r#   r.   r,   ú<module>r‚      s¾  ðð €€€Ø #Ð #Ð #Ð #Ð #Ð #Ø 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8Ø ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?à €€€Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø =Ð =Ð =Ð =Ð =Ð =Ø Ð Ð Ð Ð Ð Ø &Ð &Ð &Ð &Ð &Ð &à BÐ BÐ BÐ BÐ BÐ Bðð ð ð ð ð ð ð ð ð ð
 PÐ OÐ OÐ OÐ OÐ OÐ OÐ Oàð @Ø?Ð?Ð?Ð?Ð?Ð?à 0Ð 0Ð1MÑ NÔ NÐ Ø€gÔ˜Ñ!Ô!€ðlxð lxð lxð lxð lx ñ lxô lxð lxð^#˜EœHœOð #°ð #ð #ð #ð #ð #ð #r.   