§
    “©ºjŸ  ã                   óþ   — d dl mZ d dlmZmZmZmZ d dlZd dlm	Z	 d dlm
Z
 d dlmZ d dlmZmZ d dlZd dlmZ d d	lmZ d d
lmZmZ d dlmZ d dlmZ d dlmZ d dlm Z  erd dl!m"Z# d dl$m%Z%  G d„ de¦  «        Z&dS )é    )ÚAbstractContextManager)ÚTYPE_CHECKINGÚAnyÚCallableÚOptionalN)Úapply_to_collection)ÚTensor)ÚModule)Úget_argsÚoverride)Ú_optimizer_handles_unscaling)Ú_PRECISION_INPUT)Ú_convert_fp_tensorÚ_DtypeContextManager)Úrank_zero_warn)ÚOptimizable)Ú	Precision)ÚMisconfigurationException©ÚMixedPrecision©ÚShardedGradScalerc                   óî  ‡ — e Zd ZdZd deded         ddfd„Zededefd	„¦   «         Z	ed
e
de
ddfd„¦   «         Zed!d„¦   «         Zedefd„¦   «         Zedefd„¦   «         Zedefd„¦   «         Zede
de
fd„¦   «         Zede
de
fd„¦   «         Zededddefˆ fd„¦   «         Zededddeg e
f         de
de
f
ˆ fd„¦   «         Zedeee
f         fd„¦   «         Zedeee
f         ddfd„¦   «         Zˆ xZS )"ÚFSDPPrecisionaù  Precision plugin for training with Fully Sharded Data Parallel (FSDP).

    .. warning::  This is an :ref:`experimental <versioning:Experimental API>` feature.

    Args:
        precision: Full precision (32-true), half precision (16-true, bf16-true) or
            mixed precision (16-mixed, bf16-mixed).
        scaler: An optional :class:`torch.distributed.fsdp.sharded_grad_scaler.ShardedGradScaler` to use.

    Raises:
        ValueError:
            If unsupported ``precision`` is provided.

    NÚ	precisionÚscalerr   Úreturnc                 óŒ  — t          t          ¦  «        }||vrt          d|›d|› d�¦  «        ‚ddlm} |�!| j        dk    rt          d|›d|› d�¦  «        ‚|€|dk    r
 |¦   «         nd | _        || _        t          j        t          j        t          j	        t          j
        t          j        dœ}|| j                 | _        d S )	Nz`precision=z9)` is not supported in FSDP. `precision` must be one of: ú.r   r   ú16-mixedz` does not use a scaler, found )ú
bf16-mixedr    ú	bf16-trueú16-trueú32-true)r   r   Ú
ValueErrorÚ*torch.distributed.fsdp.sharded_grad_scalerr   r   r   ÚtorchÚfloat32Úbfloat16Úfloat16Ú_desired_input_dtype)Úselfr   r   Úsupported_precisionr   Úprecision_to_types         úx/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/pytorch_lightning/plugins/precision/fsdp.pyÚ__init__zFSDPPrecision.__init__5   s  € Ý&Õ'7Ñ8Ô8ÐØÐ/Ð/Ð/ÝðG˜ið Gð GØ0CðGð Gð Gñô ð ð
 	QÐPÐPÐPÐPÐPàÐ $¤.°JÒ">Ð">ÝÐ`¨9Ð`Ð`ÐW]Ð`Ð`Ð`ÑaÔaÐaà-3¨^À	ÈZÒ@WÐ@WÐ'Ð'Ñ)Ô)Ð)Ð]aˆŒØ"ˆŒõ  œ-ÝœÝœÝ”}Ý”}ð
ð 
Ðð %6°d´nÔ$EˆÔ!Ð!Ð!ó    Úmodulec                 óN   — d| j         v r|                     | j        ¬¦  «        S |S )NÚtrue©Údtype)r   Útor+   )r,   r2   s     r/   Úconvert_modulezFSDPPrecision.convert_moduleN   s+   € à�T”^Ð#Ð#Ø—9’9 4Ô#<�9Ñ=Ô=Ð=Øˆr1   Ú_Ú__c                 ó<   — t          d| j        j        › d�¦  «        ‚)NzA`gradient_clip_algorithm='norm'` is currently not supported for `ú`)r   Ú	__class__Ú__name__)r,   r9   r:   s      r/   Úclip_grad_by_normzFSDPPrecision.clip_grad_by_normT   s+   € õ (ØjÐPTÔP^ÔPgÐjÐjÐjñ
ô 
ð 	
r1   ÚTorchMixedPrecisionc                 ó<  — ddl m} | j        dv rt          d| j        › d�¦  «         | j        dv rt          j        x}x}}nN| j        dv rt          j        x}x}}n4| j        dk    rt          j        x}x}}nt          d	| j        ›d
�¦  «        ‚ ||||¬¦  «        S )Nr   r   )r#   r"   zFSDP with `z}` enables computation in lower precision. FSDP will always retain a full-precision copy of the model parameters for sharding.)r#   r    )r"   r!   r$   z-Was unable to infer precision type, received r   )Úparam_dtypeÚreduce_dtypeÚbuffer_dtype)	Ú2torch.distributed.fsdp.fully_sharded_data_parallelr   r   r   r'   r*   r)   r(   r   )r,   r@   rB   rC   rD   s        r/   Úmixed_precision_configz$FSDPPrecision.mixed_precision_config^   sõ   € àlÐlÐlÐlÐlÐlàŒ>Ð5Ð5Ð5Ýðf˜dœnð fð fð fñô ð ð
 Œ>Ð4Ð4Ð4Ý8=¼ÐEˆKÐE˜,¨¨ØŒ^Ð:Ð:Ð:Ý8=¼ÐFˆKÐF˜,¨¨ØŒ^˜yÒ(Ð(Ý8=¼ÐEˆKÐE˜,¨¨å+Ð,oÐ\`Ô\jÐ,oÐ,oÐ,oÑpÔpÐpà"Ð"Ø#Ø%Ø%ð
ñ 
ô 
ð 	
r1   c                 ó*   — t          | j        ¦  «        S ©N©r   r+   ©r,   s    r/   Útensor_init_contextz!FSDPPrecision.tensor_init_contextw   ó   € å# DÔ$=Ñ>Ô>Ð>r1   c                 ó*   — t          | j        ¦  «        S rH   rI   rJ   s    r/   Úmodule_init_contextz!FSDPPrecision.module_init_context{   rL   r1   c                 óª   — d| j         v r7t          j        d| j         dk    rt          j        nt          j        ¬¦  «        S t          | j        ¦  «        S )NÚmixedÚcudar!   r5   )r   r'   Úautocastr)   r*   r   r+   rJ   s    r/   Úforward_contextzFSDPPrecision.forward_context   sL   € à�d”nÐ$Ð$Ý”> &À4Ä>ÐUaÒCaÐCaµ´°ÕglÔgtÐvÑvÔvÐvÝ# DÔ$=Ñ>Ô>Ð>r1   Údatac                 óF   — t          |t          t          | j        ¬¦  «        S ©N)Úfunctionr6   Údst_type)r   r   r	   r+   ©r,   rT   s     r/   Úconvert_inputzFSDPPrecision.convert_input…   s   € å" 4Õ2DÍFÐ]aÔ]vÐwÑwÔwÐwr1   c                 ó^   — t          |t          t          t          j        ¦   «         ¬¦  «        S rV   )r   r   r	   r'   Úget_default_dtyperY   s     r/   Úconvert_outputzFSDPPrecision.convert_output‰   s$   € å" 4Õ2DÍFÕ]bÔ]tÑ]vÔ]vÐwÑwÔwÐwr1   Útensorzpl.LightningModulec                 óŠ   •— | j         �| j                              |¦  «        }t          ¦   «                              ||¦  «        S rH   )r   ÚscaleÚsuperÚpre_backward)r,   r^   r2   r=   s      €r/   rb   zFSDPPrecision.pre_backward�   s:   ø€ àŒ;Ð"Ø”[×&Ò& vÑ.Ô.ˆFÝ‰wŒw×#Ò# F¨FÑ3Ô3Ð3r1   Ú	optimizerÚmodelÚclosureÚkwargsc                 óX  •— | j         € t          ¦   «         j        |f||dœ|¤ŽS  |¦   «         }t          |¦  «        s| j                              |¦  «         |                      ||¦  «         |d u }|j        r|s. | j         j        |fi |¤Ž}| j                              ¦   «          |S |S )N)rd   re   )	r   ra   Úoptimizer_stepr   Úunscale_Ú_after_closureÚautomatic_optimizationÚstepÚupdate)	r,   rc   rd   re   rf   Úclosure_resultÚskipped_backwardÚstep_outputr=   s	           €r/   rh   zFSDPPrecision.optimizer_step“   s×   ø€ ð Œ;Ðà)•5‘7”7Ô)¨)Ð\¸5È'Ð\Ð\ÐU[Ð\Ð\Ð\Ø ˜™œˆå+¨IÑ6Ô6ð 	,ð ŒK× Ò  Ñ+Ô+Ð+à×Ò˜E 9Ñ-Ô-Ð-Ø)¨TÐ1ÐàÔ+ð 	Ð3Cð 	à*˜$œ+Ô*¨9Ð?Ð?¸Ð?Ð?ˆKØŒK×ÒÑ Ô Ð ØÐØÐr1   c                 óF   — | j         �| j                              ¦   «         S i S rH   )r   Ú
state_dictrJ   s    r/   rr   zFSDPPrecision.state_dict°   s#   € àŒ;Ð"Ø”;×)Ò)Ñ+Ô+Ð+Øˆ	r1   rr   c                 óL   — | j         �| j                              |¦  «         d S d S rH   )r   Úload_state_dict)r,   rr   s     r/   rt   zFSDPPrecision.load_state_dict¶   s.   € àŒ;Ð"ØŒK×'Ò'¨
Ñ3Ô3Ð3Ð3Ð3ð #Ð"r1   rH   )r   r@   )r>   Ú
__module__Ú__qualname__Ú__doc__r   r   r0   r   r
   r8   r   r?   ÚpropertyrF   r   rK   rN   rS   rZ   r]   r	   rb   r   r   rh   ÚdictÚstrrr   rt   Ú__classcell__)r=   s   @r/   r   r   %   sÒ  ø€ € € € € ðð ðFð FÐ"2ð F¸HÐEXÔ<Yð FÐeið Fð Fð Fð Fð2 ð Vð °ð ð ð ñ „Xðð
 ð
 Cð 
¨sð 
°tð 
ð 
ð 
ñ „Xð
ð ð
ð 
ð 
ñ „Xð
ð0 ð?Ð%;ð ?ð ?ð ?ñ „Xð?ð ð?Ð%;ð ?ð ?ð ?ñ „Xð?ð ð?Ð!7ð ?ð ?ð ?ñ „Xð?ð
 ðx #ð x¨#ð xð xð xñ „Xðxð ðx 3ð x¨3ð xð xð xñ „Xðxð ð4 6ð 4Ð3Gð 4ÈFð 4ð 4ð 4ð 4ð 4ñ „Xð4ð
 ðàðð $ðð ˜"˜c˜'Ô"ð	ð
 ðð 
ðð ð ð ð ñ „Xðð8 ð˜D  c œNð ð ð ñ „Xðð
 ð4¨$¨s°C¨x¬.ð 4¸Tð 4ð 4ð 4ñ „Xð4ð 4ð 4ð 4ð 4r1   r   )'Ú
contextlibr   Útypingr   r   r   r   r'   Úlightning_utilitiesr   r	   Útorch.nnr
   Útyping_extensionsr   r   Úpytorch_lightningÚplÚ&lightning_fabric.plugins.precision.ampr   Ú'lightning_fabric.plugins.precision.fsdpr   Ú(lightning_fabric.plugins.precision.utilsr   r   Úlightning_fabric.utilitiesr   Ú lightning_fabric.utilities.typesr   Ú-pytorch_lightning.plugins.precision.precisionr   Ú&pytorch_lightning.utilities.exceptionsr   rE   r   r@   r&   r   r   © r1   r/   ú<module>r‹      s�  ðð .Ð -Ð -Ð -Ð -Ð -Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9à €€€Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0à Ð Ð Ð Ø OÐ OÐ OÐ OÐ OÐ OØ DÐ DÐ DÐ DÐ DÐ DØ ]Ð ]Ð ]Ð ]Ð ]Ð ]Ð ]Ð ]Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø 8Ð 8Ð 8Ð 8Ð 8Ð 8Ø CÐ CÐ CÐ CÐ CÐ CØ LÐ LÐ LÐ LÐ LÐ Làð MØhÐhÐhÐhÐhÐhØLÐLÐLÐLÐLÐLðT4ð T4ð T4ð T4ð T4�Iñ T4ô T4ð T4ð T4ð T4r1   