§
    9²¸j92  ã                  ó  — d dl mZ d dlZd dlmZ d dlmZmZ ddlm	Z	m
Z
mZmZ  ej        e¦  «        Zerd dlmZ dd	lmZ  e	¦   «         rd dlZd*d„Zd+d„Zd,d„Zd,d„Zd*d„Zd-d.d„Zd„ Z	 d/d0d„Zd1d „Zd1d!„Zd2d#„Zd3d'„Z d3d(„Z!d3d)„Z"dS )4é    )ÚannotationsN)Ú	timedelta)ÚTYPE_CHECKINGÚ	TypeGuardé   )Úis_torch_availableÚis_torch_distributed_availableÚis_torch_greater_or_equalÚlogging©ÚDTensoré   )ÚDistributedConfigÚreturnÚboolc                 ó^   — t          ¦   «         sdS t          j                             ¦   «         S )NF)r	   ÚtorchÚdistributedÚis_initialized© ó    ún/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/distributed/utils.pyÚ!_is_torch_distributed_initializedr   $   s*   € Ý)Ñ+Ô+ð ØˆuÝÔ×+Ò+Ñ-Ô-Ð-r   ÚobjÚobjectúTypeGuard[DTensor]c                óN   — t          ¦   «         sdS ddlm} t          | |¦  «        S )NFr   r   )r	   Útorch.distributed.tensorr   Ú
isinstance)r   r   s     r   Ú
is_dtensorr    *   s7   € Ý)Ñ+Ô+ð ØˆuØ0Ð0Ð0Ð0Ð0Ð0å�c˜7Ñ#Ô#Ð#r   Úintc                 ó^   — t          ¦   «         sdS t          j                             ¦   «         S )Nr   )r   r   r   Úget_rankr   r   r   Ú_get_torch_distributed_rankr$   2   s*   € Ý,Ñ.Ô.ð ØˆqÝÔ×%Ò%Ñ'Ô'Ð'r   c                 ó^   — t          ¦   «         sdS t          j                             ¦   «         S )Nr   )r   r   r   Úget_world_sizer   r   r   Ú!_get_torch_distributed_world_sizer'   8   s*   € Ý,Ñ.Ô.ð ØˆqÝÔ×+Ò+Ñ-Ô-Ð-r   c                 ó€   — t          ¦   «         o0t          t          j                             dd¦  «        ¦  «        dk    S )NÚ
LOCAL_RANKz-1r   )r   r!   ÚosÚenvironÚgetr   r   r   Úis_local_dist_rank_0r-   >   s2   € Ý,Ñ.Ô.Ð_µ3µr´z·~²~ÀlÐTXÑ7YÔ7YÑ3ZÔ3ZÐ^_Ò3_Ð_r   Údevice_typeú
str | Nonec           	     ó   — t           j                             ¦   «         �s^| €#t           j                             ¦   «         j        } | dk    rt                               d¦  «         d} 	 t          t          j
        d         ¦  «        }t          t          j
        d         ¦  «        }t          t          j
        d         ¦  «        }dd	d
ddddœ}|                     | ¦  «        }d}| dk    r=t          t           | ¦  «                             |¦  «         t          j        | |¦  «        }t           j                             ||||t!          d¬¦  «        ¬¦  «         dS # t"          $ r}t%          d¦  «        |‚d}~ww xY wdS )zŽInitialize torch.distributed if not already initialized.

    If `device_type` is not given, it is detected from the current accelerator.
    NÚmpsú^PyTorch's built-in DeviceMesh/DTensor stack does not support an MPS mesh. Falling back to CPU.ÚcpuÚRANKr)   Ú
WORLD_SIZEÚncclÚglooÚxcclÚhcclÚneuronÚtpu_dist)Úcudar3   ÚxpuÚhpur:   Útpur   )Úhours)ÚbackendÚrankÚ
world_sizeÚ	device_idÚtimeoutz‘We tried to initialize torch.distributed for you, but it failed. Make sure you init torch distributed in your script to use distributed training.)r   r   r   Ú_CÚ_get_acceleratorÚtypeÚloggerÚwarning_oncer!   r*   r+   r,   ÚgetattrÚ
set_deviceÚdeviceÚinit_process_groupr   Ú	ExceptionÚOSError)r.   rB   Ú
local_rankrC   Úbackend_maprA   rD   Úes           r   Ú_ensure_torch_distributedrT   B   s¦  € õ
 Ô×+Ò+Ñ-Ô-ñ *ØÐÝœ(×3Ò3Ñ5Ô5Ô:ˆKØ˜%ÒÐÝ×ÒØpñô ð ð  ˆKð"	Ý•r”z &Ô)Ñ*Ô*ˆDÝ�RœZ¨Ô5Ñ6Ô6ˆJÝ�RœZ¨Ô5Ñ6Ô6ˆJð ØØØØ"Ø!ðð ˆKð "—o’o kÑ2Ô2ˆGð
 ˆIØ˜eÒ#Ð#Ý�˜{Ñ+Ô+×6Ò6°zÑBÔBÐBÝ!œL¨°jÑAÔA�	ÝÔ×0Ò0ØØØ%Ø#å!¨Ð*Ñ*Ô*ð 1ñ ô ð ð ð øõ ð 	ð 	ð 	Ýð^ñô ð ðøøøøð	øøøðM*ð *s   Á(C2E Å
E;Å&E6Å6E;c                 óD  — t          ¦   «         sdS t          j                             ¦   «         j        } | dk    rHt          j                             t          t          | ¦  «                             ¦   «         g¬¦  «         dS t          j                             ¦   «          dS )u3  Barrier bound to the current accelerator device.

    Passing `device_ids` is required when the process group was initialized without a
    `device_id`; with it, the call is a no-op compared to plain `barrier()`. Safe to call
    when torch.distributed has not been initialized â€” returns immediately.
    Nr3   )Ú
device_ids)	r   r   rF   rG   rH   r   ÚbarrierrK   Úcurrent_device)r.   s    r   Ú_distributed_barrierrY   t   sŒ   € õ -Ñ.Ô.ð ØˆÝ”(×+Ò+Ñ-Ô-Ô2€KØ�eÒÐÝÔ×!Ò!­gµe¸[Ñ.IÔ.I×.XÒ.XÑ.ZÔ.ZÐ-[Ð!Ñ\Ô\Ð\Ð\Ð\åÔ×!Ò!Ñ#Ô#Ð#Ð#Ð#r   Útp_planústr | dict[str, str] | NoneÚtp_sizeú
int | Nonec                ó^  — |�| €t          d¦  «        ‚| �|�t          d¦  «        ‚|�€t          d¦  «        st          d¦  «        ‚t          j                             ¦   «         j        }|dk    rt                               d¦  «         d}t          t          |¦  «        }|dk    r^| 
                    t          t          j        d	         ¦  «        ¦  «         |                     ¦   «         }t          j        ||¦  «        }|}nt          j        |¦  «        }|pi }t          j                             |j        |f¦  «        }nf|j        d
k    r d|j        vrt          d¦  «        ‚|d         }t          j        |j        › dt          t          j        d	         ¦  «        › �¦  «        }||fS )z¬
    Sets up the device mesh and initialized the backend for tensor parallelism.
    This function is called when the model is loaded and the TP plan is set to 'auto'.
    Nz-tp_plan has to be set when tp_size is passed.zY`tp_plan` and `device_map` are mutually exclusive. Choose either one for parallelization.ú2.5z3Tensor parallel is only supported for `torch>=2.5`.r1   r2   r3   r)   r   ÚtpzsWhen using `tp_plan` and n-d `device_mesh`, it must contain a 'tp' dimension. Please provide a valid `device_mesh`.ú:)Ú
ValueErrorr
   rP   r   rF   rG   rH   rI   rJ   rK   rL   r!   r*   r+   rX   rM   r   Úinit_device_meshÚndimÚmesh_dim_namesr.   )rZ   r\   Údevice_meshÚ
device_mapr.   rX   ÚindexÚ	tp_devices           r   Úinitialize_tensor_parallelismrj   …   s¸  € ð Ð˜w˜ÝÐHÑIÔIÐIØÐ˜zÐ5ÝÐtÑuÔuÐuØÑÝ(¨Ñ/Ô/ð 	QÝÐOÑPÔPÐPõ ”h×/Ò/Ñ1Ô1Ô6ˆØ˜%ÒÐÝ×ÒØpñô ð ð  ˆKÝ ¥¨Ñ4Ô4ˆà˜%ÒÐØ×%Ò%¥c­"¬*°\Ô*BÑ&CÔ&CÑDÔDÐDØ"×1Ò1Ñ3Ô3ˆEÝœ [°%Ñ8Ô8ˆIØ"ˆJˆJåœ [Ñ1Ô1ˆIØ$Ð*¨ˆJåÔ'×8Ò8¸¼È'ÈÑTÔTˆˆàÔ˜aÒÐØ˜;Ô5Ð5Ð5Ý ð<ñô ð ð & dÔ+ˆKÝ”\ [Ô%<Ð"^Ð"^½sÅ2Ä:ÈlÔC[Ñ?\Ô?\Ð"^Ð"^Ñ_Ô_ˆ
à�{Ð"Ð"r   Údistributed_configr   c                ó  — | j         dk    rt          d¦  «        st          d¦  «        ‚t          j                             ¦   «         j        }|dk    rkt          t          j	         
                    dd¦  «        ¦  «        }t          t          |¦  «                             |¦  «         t          j        ||¦  «        }nt          j        |¦  «        }| j         }g g }}|dk    r*|                     |¦  «         |                     d¦  «         t          j                             |t#          |¦  «        t#          |¦  «        ¬¦  «        }t%          |¦  «        dk    r(|                     d	                     |¦  «        ¦  «         ||fS )
Nr   ú2.7z?FSDP2 requires `torch>=2.7` (distributed checkpoint save/load).r3   r)   r   Úfsdp©re   Ú_)Ú	fsdp_sizer
   rP   r   rF   rG   rH   r!   r*   r+   r,   rK   rL   rM   Úappendr   rc   ÚtupleÚlenÚ_flattenÚjoin)rk   r.   rQ   rg   rq   ÚdimsÚnamesÚmeshs           r   Ú)initialize_fully_sharded_data_parallelismrz   ´   sP  € ð Ô# aÒ'Ð'Õ0IÈ%Ñ0PÔ0PÐ'ÝÐWÑXÔXÐXå”(×+Ò+Ñ-Ô-Ô2€Kà�eÒÐÝ�œŸš¨°aÑ8Ô8Ñ9Ô9ˆ
Ý•�{Ñ#Ô#×.Ò.¨zÑ:Ô:Ð:Ý”\ +¨zÑ:Ô:ˆ
ˆ
å”\ +Ñ.Ô.ˆ
à"Ô,€Ià�bˆ%€DØ�1‚}€}Ø�Š�IÑÔÐØ�Š�VÑÔÐõ Ô×-Ò-¨k½5À¹;¼;ÕW\Ð]bÑWcÔWcÐ-ÑdÔd€Då
ˆ4�y„y�1‚}€}Ø�Š�c—h’h˜u‘o”oÑ&Ô&Ð&à�tÐÐr   c                óª  — t          d¦  «        st          d¦  «        ‚t          j                             ¦   «         j        }t          |¦  «         t          j                             ¦   «         }| j	        }||k    rt          d|› d|› d�¦  «        ‚|dk    rkt          t          j                             dd¦  «        ¦  «        }t          t          |¦  «                             |¦  «         t          j        ||¦  «        }nt          j        |¦  «        }||k    sJ d|› d|› d�¦   «         ‚t          j                             ||fd	¬
¦  «        }||fS )Nr_   zBPipeline parallelism with DistributedConfig requires `torch>=2.5`.zworld_size (z) must be equal to pp_size (ú)r3   r)   r   )Úppro   )r
   rP   r   rF   rG   rH   rT   r   r&   Úpp_sizeÚRuntimeErrorr!   r*   r+   r,   rK   rL   rM   rc   )rk   r.   rC   r~   rQ   rg   ry   s          r   Úinitialize_pipeline_parallelismr€   Ó   sS  € õ % UÑ+Ô+ð \ÝÐZÑ[Ô[Ð[å”(×+Ò+Ñ-Ô-Ô2€KÝ˜kÑ*Ô*Ð*åÔ"×1Ò1Ñ3Ô3€JØ Ô(€GØ�WÒÐÝÐ\¨*Ð\Ð\ÐRYÐ\Ð\Ð\Ñ]Ô]Ð]à�eÒÐÝ�œŸš¨°aÑ8Ô8Ñ9Ô9ˆ
Ý•�{Ñ#Ô#×.Ò.¨zÑ:Ô:Ð:Ý”\ +¨zÑ:Ô:ˆ
ˆ
å”\ +Ñ.Ô.ˆ
à˜Ò Ð Ð Ð"c°Ð"cÐ"cÐY`Ð"cÐ"cÐ"cÑ Ô Ð ÝÔ×-Ò-¨k¸G¸:ÐV]Ð-Ñ^Ô^€Dà�tÐÐr   údict[str, torch.Tensor]c                ó®   — t          d¦  «        st          d¦  «        ‚ddlm}m}  |dd¬¦  «        } || |¬¦  «        }t          ¦   «         dk    r|S i S )z~Gather FSDP-sharded params to full plain CPU tensors.

    Only rank 0 accumulates the result; other ranks return ``{}``.
    rm   ú0Distributed checkpointing requires `torch>=2.7`.r   )ÚStateDictOptionsÚget_model_state_dictT)Úfull_state_dictÚcpu_offload)Úoptions)r
   rP   Ú'torch.distributed.checkpoint.state_dictr„   r…   r$   )Úmodelr„   r…   rˆ   r†   s        r   Úgather_full_state_dictr‹   î   s…   € õ
 % UÑ+Ô+ð JÝÐHÑIÔIÐIð _Ð^Ð^Ð^Ð^Ð^Ð^Ð^àÐ¨tÀÐFÑFÔF€GØ*Ð*¨5¸'ÐBÑBÔB€OÝ"Ñ$Ô$¨Ò)Ð)ØÐØ€Ir   Úcheckpoint_dirÚstrÚNonec                óä   — t          d¦  «        st          d¦  «        ‚ddlmc m} ddlm} ddlm}  || ¦  «        }| 	                    | ||dd¬¦  «        ¬	¦  «         t          ¦   «          dS )
uè  Save model parameters as standard HF-format sharded safetensors using
    DCP + HuggingFaceStorageWriter with consolidation enabled.

    Every rank first writes its own shard in parallel under
    `<checkpoint_dir>/sharded/`, then a consolidation pass reads those shards
    and emits HF-compatible `model-*-of-N.safetensors` (+ index) at
    `<checkpoint_dir>/`. The result is a directory `from_pretrained` reads
    through its normal path â€” no special flag needed at load time.
    rm   rƒ   r   N)ÚHuggingFaceStorageWriter)r…   T)ÚpathÚsave_distributedÚenable_consolidation)Ústorage_writer)r
   rP   Útorch.distributed.checkpointr   Ú
checkpointÚ'torch.distributed.checkpoint.hf_storager�   r‰   r…   ÚsaverY   )rŠ   rŒ   Údcpr�   r…   Ú
state_dicts         r   Ú!save_model_checkpoint_distributedr›     sÅ   € õ % UÑ+Ô+ð JÝÐHÑIÔIÐIð /Ð.Ð.Ð.Ð.Ð.Ð.Ð.Ð.ØPÐPÐPÐPÐPÐPØLÐLÐLÐLÐLÐLà%Ð% eÑ,Ô,€JØ‡H‚HØØ/Ð/ØØ!Ø!%ð
ñ 
ô 
ð ñ ô ð õ ÑÔÐÐÐr   c                óª   — t          d¦  «        st          d¦  «        ‚ddlmc m} ddlm}  || |¦  «        }|                     d|i|¬¦  «         dS )zSave optimizer state via DCP.rm   rƒ   r   N)Úget_optimizer_state_dictÚ	optimizer©Úcheckpoint_id)r
   rP   r•   r   r–   r‰   r�   r˜   )rŠ   rž   rŒ   r™   r�   Úoptimizer_state_dicts         r   Úsave_optimizer_distributedr¢   "  s†   € å$ UÑ+Ô+ð JÝÐHÑIÔIÐIð /Ð.Ð.Ð.Ð.Ð.Ð.Ð.Ð.ØPÐPÐPÐPÐPÐPà3Ð3°E¸9ÑEÔEÐØ‡H‚HˆkÐ/Ð0À€HÑOÔOÐOÐOÐOr   c                óÈ   — t          d¦  «        st          d¦  «        ‚ddlmc m} ddlm}m}  || |¦  «        }|                     d|i|¬¦  «          || ||¦  «         dS )zLoad optimizer state via DCP.rm   rƒ   r   N)r�   Úset_optimizer_state_dictrž   rŸ   )	r
   rP   r•   r   r–   r‰   r�   r¤   Úload)rŠ   rž   rŒ   r™   r�   r¤   r¡   s          r   Úload_optimizer_distributedr¦   0  s¢   € å$ UÑ+Ô+ð JÝÐHÑIÔIÐIð /Ð.Ð.Ð.Ð.Ð.Ð.Ð.Ð.ØjÐjÐjÐjÐjÐjÐjÐjà3Ð3°E¸9ÑEÔEÐØ‡H‚HˆkÐ/Ð0À€HÑOÔOÐOØÐ˜U IÐ/CÑDÔDÐDÐDÐDr   )r   r   )r   r   r   r   )r   r!   )N)r.   r/   )NNN)rZ   r[   r\   r]   )rk   r   )r   r�   )rŒ   r�   r   rŽ   )#Ú
__future__r   r*   Údatetimer   Útypingr   r   Úutilsr   r	   r
   r   Ú
get_loggerÚ__name__rI   r   r   Úconfiguration_utilsr   r   r   r    r$   r'   r-   rT   rY   rj   rz   r€   r‹   r›   r¢   r¦   r   r   r   ú<module>r®      s  ðð #Ð "Ð "Ð "Ð "Ð "à 	€	€	€	Ø Ð Ð Ð Ð Ð Ø +Ð +Ð +Ð +Ð +Ð +Ð +Ð +à jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jð 
ˆÔ	˜HÑ	%Ô	%€ð ð 7Ø0Ð0Ð0Ð0Ð0Ð0à6Ð6Ð6Ð6Ð6Ð6ð ÐÑÔð Ø€L€L€Lð.ð .ð .ð .ð$ð $ð $ð $ð(ð (ð (ð (ð.ð .ð .ð .ð`ð `ð `ð `ð/ð /ð /ð /ð /ðd$ð $ð $ð$ dhð,#ð ,#ð ,#ð ,#ð ,#ð^ð ð ð ð>ð ð ð ð6ð ð ð ð&ð ð ð ðBPð Pð Pð PðEð Eð Eð Eð Eð Er   