§
    “©ºj¾  ã                   óú   — d dl mZ d dlmZmZmZmZ d dlZd dlm	Z	 d dlm
Z
 d dlmZ d dlmZ d dlmZmZ d d	lmZ d d
lmZ d dlmZmZ d dlmZ d dlmZ erd dlmZ d dl m!Z! ed         Z" G d„ de¦  «        Z#dS )é    )ÚAbstractContextManager)ÚTYPE_CHECKINGÚAnyÚLiteralÚOptionalN)Úapply_to_collection)ÚTensor)ÚModule)Ú	Optimizer)Úget_argsÚoverride)Ú_optimizer_handles_unscaling)Ú	Precision)Ú_convert_fp_tensorÚ_DtypeContextManager)Úrank_zero_warn)ÚOptimizable©ÚMixedPrecision©ÚShardedGradScaler)ú32-trueú16-trueú	bf16-trueú16-mixedú
bf16-mixedc                   óæ  ‡ — e Zd ZdZddeded         ddfd„Zededefd	„¦   «         Z	e
dd„¦   «         Zedefd„¦   «         Zedefd„¦   «         Zedefd„¦   «         Zededefd„¦   «         Zededefd„¦   «         Zededee         dededdf
ˆ fd„¦   «         Zedededefˆ fd„¦   «         Zededdfd„¦   «         Zedeeef         fd„¦   «         Zedeeef         ddfd„¦   «         Zˆ xZS )ÚFSDPPrecisionaù  Precision plugin for training with Fully Sharded Data Parallel (FSDP).

    .. warning::  This is an :ref:`experimental <versioning:Experimental API>` feature.

    Args:
        precision: Full precision (32-true), half precision (16-true, bf16-true) or
            mixed precision (16-mixed, bf16-mixed).
        scaler: An optional :class:`torch.distributed.fsdp.sharded_grad_scaler.ShardedGradScaler` to use.

    Raises:
        ValueError:
            If unsupported ``precision`` is provided.

    NÚ	precisionÚscalerr   Úreturnc                 óŒ  — t          t          ¦  «        }||vrt          d|›d|› d�¦  «        ‚ddlm} |�!| j        dk    rt          d|›d|› d�¦  «        ‚|€|dk    r
 |¦   «         nd | _        || _        t          j        t          j        t          j	        t          j
        t          j        dœ}|| j                 | _        d S )	Nz`precision=z9)` is not supported in FSDP. `precision` must be one of: ú.r   r   r   z` does not use a scaler, found )r   r   r   r   r   )r   Ú_PRECISION_INPUTÚ
ValueErrorÚ*torch.distributed.fsdp.sharded_grad_scalerr   r   r    ÚtorchÚfloat32Úbfloat16Úfloat16Ú_desired_input_dtype)Úselfr   r    Úsupported_precisionr   Úprecision_to_types         úw/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/lightning_fabric/plugins/precision/fsdp.pyÚ__init__zFSDPPrecision.__init__5   s  € Ý&Õ'7Ñ8Ô8ÐØÐ/Ð/Ð/ÝðG˜ið Gð GØ0CðGð Gð Gñô ð ð
 	QÐPÐPÐPÐPÐPàÐ $¤.°JÒ">Ð">ÝÐ`¨9Ð`Ð`ÐW]Ð`Ð`Ð`ÑaÔaÐaà-3¨^À	ÈZÒ@WÐ@WÐ'Ð'Ñ)Ô)Ð)Ð]aˆŒØ"ˆŒõ  œ-ÝœÝœÝ”}Ý”}ð
ð 
Ðð %6°d´nÔ$EˆÔ!Ð!Ð!ó    Úmodulec                 óN   — d| j         v r|                     | j        ¬¦  «        S |S )NÚtrue©Údtype)r   Útor+   )r,   r2   s     r/   Úconvert_modulezFSDPPrecision.convert_moduleN   s+   € à�T”^Ð#Ð#Ø—9’9 4Ô#<�9Ñ=Ô=Ð=Øˆr1   ÚTorchMixedPrecisionc                 ó<  — ddl m} | j        dv rt          d| j        › d�¦  «         | j        dv rt          j        x}x}}nN| j        dv rt          j        x}x}}n4| j        dk    rt          j        x}x}}nt          d	| j        ›d
�¦  «        ‚ ||||¬¦  «        S )Nr   r   )r   r   zFSDP with `z}` enables computation in lower precision. FSDP will always retain a full-precision copy of the model parameters for sharding.)r   r   )r   r   r   z-Was unable to infer precision type, received r#   )Úparam_dtypeÚreduce_dtypeÚbuffer_dtype)	Ú2torch.distributed.fsdp.fully_sharded_data_parallelr   r   r   r'   r*   r)   r(   r%   )r,   r9   r;   r<   r=   s        r/   Úmixed_precision_configz$FSDPPrecision.mixed_precision_configT   só   € àlÐlÐlÐlÐlÐlàŒ>Ð5Ð5Ð5Ýðf˜dœnð fð fð fñô ð ð
 Œ>Ð4Ð4Ð4Ý8=¼ÐEˆKÐE˜,¨¨ØŒ^Ð:Ð:Ð:Ý8=¼ÐFˆKÐF˜,¨¨ØŒ^˜yÒ(Ð(Ý8=¼ÐEˆKÐE˜,¨¨åÐ`ÈTÌ^Ð`Ð`Ð`ÑaÔaÐaà"Ð"Ø#Ø%Ø%ð
ñ 
ô 
ð 	
r1   c                 ó*   — t          | j        ¦  «        S ©N©r   r+   ©r,   s    r/   Útensor_init_contextz!FSDPPrecision.tensor_init_contextm   ó   € å# DÔ$=Ñ>Ô>Ð>r1   c                 ó*   — t          | j        ¦  «        S rA   rB   rC   s    r/   Úmodule_init_contextz!FSDPPrecision.module_init_contextq   rE   r1   c                 óª   — d| j         v r7t          j        d| j         dk    rt          j        nt          j        ¬¦  «        S |                      ¦   «         S )NÚmixedÚcudar   r5   )r   r'   Úautocastr)   r*   rD   rC   s    r/   Úforward_contextzFSDPPrecision.forward_contextu   sM   € à�d”nÐ$Ð$Ý”> &À4Ä>ÐUaÒCaÐCaµ´°ÕglÔgtÐvÑvÔvÐvØ×'Ò'Ñ)Ô)Ð)r1   Údatac                 óF   — t          |t          t          | j        ¬¦  «        S ©N)Úfunctionr6   Údst_type)r   r   r	   r+   ©r,   rM   s     r/   Úconvert_inputzFSDPPrecision.convert_input{   s   € å" 4Õ2DÍFÐ]aÔ]vÐwÑwÔwÐwr1   c                 ó^   — t          |t          t          t          j        ¦   «         ¬¦  «        S rO   )r   r   r	   r'   Úget_default_dtyperR   s     r/   Úconvert_outputzFSDPPrecision.convert_output   s$   € å" 4Õ2DÍFÕ]bÔ]tÑ]vÔ]vÐwÑwÔwÐwr1   ÚtensorÚmodelÚargsÚkwargsc                 ó†   •— | j         �| j                              |¦  «        } t          ¦   «         j        ||g|¢R i |¤Ž d S rA   )r    ÚscaleÚsuperÚbackward)r,   rW   rX   rY   rZ   Ú	__class__s        €r/   r^   zFSDPPrecision.backwardƒ   sO   ø€ àŒ;Ð"Ø”[×&Ò& vÑ.Ô.ˆFØ�‰ŒÔ˜ Ð8¨Ð8Ð8Ð8°Ð8Ð8Ð8Ð8Ð8r1   Ú	optimizerc                 ó¢   •— | j         € t          ¦   «         j        |fi |¤ŽS  | j         j        |fi |¤Ž}| j                              ¦   «          |S rA   )r    r]   Úoptimizer_stepÚstepÚupdate)r,   r`   rZ   Ústep_outputr_   s       €r/   rb   zFSDPPrecision.optimizer_step‰   sb   ø€ ð Œ;Ðà)•5‘7”7Ô)¨)Ð>Ð>°vÐ>Ð>Ð>à&�d”kÔ& yÐ;Ð;°FÐ;Ð;ˆØŒ×ÒÑÔÐØÐr1   c                 ó‚   — | j         }|�5t          |¦  «        rt          d¦  «        ‚|                     |¦  «         d S d S )NzKGradient clipping is not implemented for optimizers handling the unscaling.)r    r   ÚNotImplementedErrorÚunscale_)r,   r`   r    s      r/   Úunscale_gradientszFSDPPrecision.unscale_gradients—   sO   € à”ˆØÐÝ+¨IÑ6Ô6ð yÝ)Ð*wÑxÔxÐxØ�OŠO˜IÑ&Ô&Ð&Ð&Ð&ð Ðr1   c                 óF   — | j         �| j                              ¦   «         S i S rA   )r    Ú
state_dictrC   s    r/   rk   zFSDPPrecision.state_dictŸ   s#   € àŒ;Ð"Ø”;×)Ò)Ñ+Ô+Ð+Øˆ	r1   rk   c                 óL   — | j         �| j                              |¦  «         d S d S rA   )r    Úload_state_dict)r,   rk   s     r/   rm   zFSDPPrecision.load_state_dict¥   s.   € àŒ;Ð"ØŒK×'Ò'¨
Ñ3Ô3Ð3Ð3Ð3ð #Ð"r1   rA   )r!   r9   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r$   r   r0   r   r
   r8   Úpropertyr?   r   rD   rG   rL   r   rS   rV   r	   r^   r   rb   r   ri   ÚdictÚstrrk   rm   Ú__classcell__)r_   s   @r/   r   r   %   sÂ  ø€ € € € € ðð ðFð FÐ"2ð F¸HÐEXÔ<Yð FÐeið Fð Fð Fð Fð2 ð Vð °ð ð ð ñ „Xðð
 ð
ð 
ð 
ñ „Xð
ð0 ð?Ð%;ð ?ð ?ð ?ñ „Xð?ð ð?Ð%;ð ?ð ?ð ?ñ „Xð?ð ð*Ð!7ð *ð *ð *ñ „Xð*ð
 ðx #ð x¨#ð xð xð xñ „Xðxð ðx 3ð x¨3ð xð xð xñ „Xðxð ð9˜vð 9¨h°vÔ.>ð 9Àsð 9ÐVYð 9Ð^bð 9ð 9ð 9ð 9ð 9ñ „Xð9ð
 ðàðð ðð 
ð	ð ð ð ð ñ „Xðð ð'¨9ð '¸ð 'ð 'ð 'ñ „Xð'ð ð˜D  c œNð ð ð ñ „Xðð
 ð4¨$¨s°C¨x¬.ð 4¸Tð 4ð 4ð 4ñ „Xð4ð 4ð 4ð 4ð 4r1   r   )$Ú
contextlibr   Útypingr   r   r   r   r'   Úlightning_utilitiesr   r	   Útorch.nnr
   Útorch.optimr   Útyping_extensionsr   r   Ú&lightning_fabric.plugins.precision.ampr   Ú,lightning_fabric.plugins.precision.precisionr   Ú(lightning_fabric.plugins.precision.utilsr   r   Úlightning_fabric.utilitiesr   Ú lightning_fabric.utilities.typesr   r>   r   r9   r&   r   r$   r   © r1   r/   ú<module>r‚      s}  ðð .Ð -Ð -Ð -Ð -Ð -Ø 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8à €€€Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø !Ð !Ð !Ð !Ð !Ð !Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0à OÐ OÐ OÐ OÐ OÐ OØ BÐ BÐ BÐ BÐ BÐ BØ ]Ð ]Ð ]Ð ]Ð ]Ð ]Ð ]Ð ]Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø 8Ð 8Ð 8Ð 8Ð 8Ð 8àð MØhÐhÐhÐhÐhÐhØLÐLÐLÐLÐLÐLàÐVÔWÐ ðC4ð C4ð C4ð C4ð C4�Iñ C4ô C4ð C4ð C4ð C4r1   