§
    9²¸jÑ.  ã                  ó  — d dl mZ d dlZd dlZd dlZd dlmZ ddlmZm	Z	 ddl
mZ ddlmZ dd	lmZmZ dd
lmZ ddlmZmZmZ ddlmZmZmZmZmZmZmZmZmZm Z   e	j!        e"¦  «        Z#erd dl$m%Z%  G d„ d¦  «        Z&dS )é    )ÚannotationsN)ÚTYPE_CHECKINGé   )Úis_torch_greater_or_equalÚlogging)Úcreate_and_tag_model_cardé   )ÚDistributedConfig)Ú$apply_fully_sharded_data_parallelismÚis_fsdp_managed_module)Úapply_pipeline_parallelism)Ú_validate_tp_plan_stylesÚapply_tensor_parallelismÚgather_state_dict_for_save)
Ú_distributed_barrierÚ_ensure_torch_distributedÚ_get_torch_distributed_rankÚ!_get_torch_distributed_world_sizeÚ!_is_torch_distributed_initializedÚgather_full_state_dictÚ)initialize_fully_sharded_data_parallelismÚinitialize_pipeline_parallelismÚinitialize_tensor_parallelismÚ!save_model_checkpoint_distributedc                  óX  — e Zd ZU dZdZdZded<   dZded<   dZdZ	ded<   dZ
ded<   d5d„Zed6d„¦   «         Zed6d„¦   «         Zed7d„¦   «         Zej        d8d„¦   «         Zej        d9d„¦   «         Zedddœd:d„¦   «         Zed;d„¦   «         Zd<d„Zd d!ddddd d"œd=d/„Zd!d0œd>d3„Zd?d4„ZdS )@ÚDistributedMixinzFDistributed orchestration and save/load hooks for [`PreTrainedModel`].Núdict[str, str] | NoneÚ_tp_planÚ_ep_planú!dict[str, tuple[str, str]] | NoneÚ_pp_planÚ
_fsdp_planÚreturnÚNonec                ó°  ‡— t          | ¦  «        }t          t          |dd¦  «        pi ¦  «        | _        t          t          |dd¦  «        pi ¦  «        | _        t          t          |dd¦  «        pi ¦  «        | _        t          t          |dd¦  «        pi ¦  «        | _        | j        | u r˜| j                             | j	        j
        pi ¦  «         | j                             | j	        j        pi ¦  «         | j                             | j	        j        pi ¦  «         | j                             | j	        j        pi ¦  «         |                      ¦   «         D �]z\  Š}t          |dd¦  «        x}rJ| j                             ˆfd„|                     ¦   «                              ¦   «         D ¦   «         ¦  «         t          |dd¦  «        x}rJ| j                             ˆfd„|                     ¦   «                              ¦   «         D ¦   «         ¦  «         t          |dd¦  «        x}rJ| j                             ˆfd„|                     ¦   «                              ¦   «         D ¦   «         ¦  «         t          |dd¦  «        x}rJ| j                             ˆfd	„|                     ¦   «                              ¦   «         D ¦   «         ¦  «         �Œ|dS )
zQCopy class-level plans onto the instance and merge config/children contributions.r   Nr   r!   r"   c                ó&   •— i | ]\  }}‰› d |› �|“ŒS ©ú.© ©Ú.0ÚkÚvÚnames      €ún/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/distributed/mixin.pyú
<dictcomp>z8DistributedMixin.init_parallel_plans.<locals>.<dictcomp>N   ó)   ø€ Ð%WÐ%WÐ%W¹4¸1¸a¨ m m° m m°QÐ%WÐ%WÐ%Wó    c                ó&   •— i | ]\  }}‰› d |› �|“ŒS r'   r)   r*   s      €r/   r0   z8DistributedMixin.init_parallel_plans.<locals>.<dictcomp>P   r1   r2   c                ó&   •— i | ]\  }}‰› d |› �|“ŒS r'   r)   r*   s      €r/   r0   z8DistributedMixin.init_parallel_plans.<locals>.<dictcomp>R   r1   r2   c                ó&   •— i | ]\  }}‰› d |› �|“ŒS r'   r)   r*   s      €r/   r0   z8DistributedMixin.init_parallel_plans.<locals>.<dictcomp>T   s)   ø€ Ð'YÐ'YÐ'Y¹T¸QÀ¨4¨¨°!¨¨°qÐ'YÐ'YÐ'Yr2   )ÚtypeÚdictÚgetattrr   r   r!   r"   Ú
base_modelÚupdateÚconfigÚbase_model_pp_planÚbase_model_tp_planÚbase_model_ep_planÚbase_model_fsdp_planÚnamed_childrenÚcopyÚitems)ÚselfÚ	model_clsÚmoduleÚplanr.   s       @r/   Úinit_parallel_plansz$DistributedMixin.init_parallel_plans>   s­  ø€ å˜‘J”Jˆ	Ý�W Y°
¸DÑAÔAÐGÀRÑHÔHˆŒÝ�W Y°
¸DÑAÔAÐGÀRÑHÔHˆŒÝ�W Y°
¸DÑAÔAÐGÀRÑHÔHˆŒÝ�w y°,ÀÑEÔEÐKÈÑLÔLˆŒàŒ?˜dÐ"Ð"ØŒM× Ò  ¤Ô!?Ð!EÀ2ÑFÔFÐFØŒM× Ò  ¤Ô!?Ð!EÀ2ÑFÔFÐFØŒM× Ò  ¤Ô!?Ð!EÀ2ÑFÔFÐFØŒO×"Ò" 4¤;Ô#CÐ#IÀrÑJÔJÐJà ×/Ò/Ñ1Ô1ð 	[ñ 	[‰LˆD�&Ý˜v z°4Ñ8Ô8Ð8ˆtð YØ”×$Ò$Ð%WÐ%WÐ%WÐ%WÀ4Ç9Â9Á;Ä;×CTÒCTÑCVÔCVÐ%WÑ%WÔ%WÑXÔXÐXÝ˜v z°4Ñ8Ô8Ð8ˆtð YØ”×$Ò$Ð%WÐ%WÐ%WÐ%WÀ4Ç9Â9Á;Ä;×CTÒCTÑCVÔCVÐ%WÑ%WÔ%WÑXÔXÐXÝ˜v z°4Ñ8Ô8Ð8ˆtð YØ”×$Ò$Ð%WÐ%WÐ%WÐ%WÀ4Ç9Â9Á;Ä;×CTÒCTÑCVÔCVÐ%WÑ%WÔ%WÑXÔXÐXÝ˜v |°TÑ:Ô:Ð:ˆtð [Ø”×&Ò&Ð'YÐ'YÐ'YÐ'YÀTÇYÂYÁ[Ä[×EVÒEVÑEXÔEXÐ'YÑ'YÔ'YÑZÔZÐZùð	[ð 	[r2   údict[str, str]c                ó²   — t          | j        d¦  «        r<| j        j        j        r+| j        st          d| j        j        › d�¦  «        ‚| j        S | j        S )z)The full tp plan for the model's modules.Údistributed_configzGExpert parallelism was requested (`enable_expert_parallel=True`), but `zs` does not define an expert-parallel plan. Add a `base_model_ep_plan` to its config, or disable expert parallelism.)	Úhasattrr;   rJ   Úenable_expert_parallelr   Ú
ValueErrorÚ	__class__Ú__name__r   ©rC   s    r/   Útp_planzDistributedMixin.tp_planV   sv   € õ �4”;Ð 4Ñ5Ô5ð 	!¸$¼+Ô:XÔ:oð 	!Ø”=ð Ý ðZØœÔ/ðZð Zð Zñô ð ð
 ”=Ð ØŒ}Ðr2   c                ó   — | j         S ©N)r"   rP   s    r/   Ú	fsdp_planzDistributedMixin.fsdp_planc   s
   € àŒÐr2   údict[str, tuple[str, str]]c                ó   — | j         S rS   )r!   rP   s    r/   Úpp_planzDistributedMixin.pp_plang   s
   € àŒ}Ðr2   rF   c                óš  — |€	i | _         d S t          |t          ¦  «        st          d¦  «        ‚t	          |¦  «         d„ |                      ¦   «         D ¦   «         }|                     ¦   «         D ]R}|                     dd¦  «        }d}|D ]}t          j	        ||¦  «        rd} nŒ|st          j        d|› d�¦  «         ŒS|| _         d S )	Nz&Can only set a dictionary as `tp_plan`c                ó   — g | ]\  }}|‘ŒS r)   r)   )r+   r.   Ú_s      r/   ú
<listcomp>z,DistributedMixin.tp_plan.<locals>.<listcomp>u   s   € ÐIÐIÐI¡g d¨A˜TÐIÐIÐIr2   Ú*z\d+FTzLayer pattern 'z�' does not match any parameters in the model. This rule may not be applied during tensor parallelization, or may lead to dimension mismatches)r   Ú
isinstancer7   rM   r   Únamed_parametersÚkeysÚreplaceÚreÚmatchÚwarningsÚwarn)rC   rF   Úmodel_param_namesÚlayer_patternÚregex_patternÚpattern_matchedÚ
param_names          r/   rQ   zDistributedMixin.tp_plank   s  € àˆ<ØˆDŒMØˆFÝ˜$¥Ñ%Ô%ð 	GÝÐEÑFÔFÐFå  Ñ&Ô&Ð&àIÐI°×1FÒ1FÑ1HÔ1HÐIÑIÔIÐØ!ŸYšY™[œ[ð 	ð 	ˆMØ)×1Ò1°#°vÑ>Ô>ˆMØ#ˆOØ/ð ð �
Ý”8˜M¨:Ñ6Ô6ð Ø&*�OØ�Eðð #ð Ý”ðd mð dð dð dñô ð øð
 ˆŒˆˆr2   c                ór   — |€	i | _         d S t          |t          ¦  «        st          d¦  «        ‚|| _         d S )Nz&Can only set a dictionary as `pp_plan`)r!   r]   r7   rM   )rC   rF   s     r/   rW   zDistributedMixin.pp_plan…   s@   € àˆ<ØˆDŒMØˆFÝ˜$¥Ñ%Ô%ð 	GÝÐEÑFÔFÐFàˆŒˆˆr2   )Údevice_meshÚ
device_maprJ   úDistributedConfig | dict | Noneú/tuple[DistributedConfig | None, object, object]c          	     ó&  — |€d ||fS t          |t          ¦  «        rt          j        |¦  «        }|j        dk    s|j        dk    rRt          ¦   «          t          ¦   «         }|j        |j        z  |k    r#t          d|j        › d|j        › d|› d�¦  «        ‚|j        dk    r/|j	        €d|_	        t          |j	        |j        ||¬¦  «        \  }}n;|j        dk    rt          |¦  «        \  }}n|j        dk    rt          |¦  «        \  }}|||fS )Nr	   z	tp_size (z) * fsdp_size (z) is not equal to world_size (ú)Úauto)Útp_sizerk   rl   )r]   r7   r
   Ú	from_dictrr   Ú	fsdp_sizer   r   ÚRuntimeErrorrQ   r   r   Úpp_sizer   )ÚclsrJ   rk   rl   Ú
world_sizes        r/   Úprepare_distribute_modelz)DistributedMixin.prepare_distribute_model�   sz  € ð Ð%Ø˜ [Ð0Ð0åÐ(­$Ñ/Ô/ð 	QÝ!2Ô!<Ð=OÑ!PÔ!PÐàÔ%¨Ò)Ð)Ð-?Ô-IÈAÒ-MÐ-MÝ%Ñ'Ô'Ð'Ý:Ñ<Ô<ˆJØ!Ô)Ð,>Ô,HÑHÈJÒVÐVÝ"ðAÐ 2Ô :ð Að AÐK]ÔKgð Að AØ3=ðAð Að Añô ð ð
 Ô%¨Ò)Ð)Ø!Ô)Ð1Ø-3Ð"Ô*Ý&CØ"Ô*Ø*Ô2Ø'Ø%ð	'ñ 'ô 'Ñ#ˆJ˜˜ð  Ô)¨AÒ-Ð-Ý&OÐPbÑ&cÔ&cÑ#ˆJ˜˜ØÔ'¨!Ò+Ð+Ý&EÐFXÑ&YÔ&YÑ#ˆJ˜à! :¨{Ð:Ð:r2   Úmodelú	nn.ModuleúDistributedConfig | Nonec                ó¸  — |�×||j         _        ||_        |j        |_        |j        dk    rL|j        dk    r|d         n|}t          |j        t          ¦  «        r|j        |_        t          ||¦  «        }na|j
        dk    r&|j        dk    r|d         n|}t          ||¦  «        }n0|j        dk    r%|j        dk    r|d         n|}t          ||¦  «        }|S )z:Apply TP or FSDP2 after model init, before weight loading.Nr	   ÚtpÚfsdpÚpp)r;   rJ   Ú_device_meshrr   Ú_tp_sizeÚndimr]   rQ   r7   r   rt   r   rv   r   )rw   rz   rJ   rk   Útp_meshÚ	fsdp_meshÚpp_meshs          r/   Úmaybe_distribute_modelz'DistributedMixin.maybe_distribute_model¶   sú   € ð Ð"Ø.@ˆEŒLÔ+Ø!,ˆEÔØ/Ô7ˆEŒNà!Ô)¨AÒ-Ð-Ø/:Ô/?À!Ò/CÐ/C˜+ dÔ+Ð+È�ÝÐ0Ô8½$Ñ?Ô?ð ?Ø$6Ô$>�E”MÝ0°¸Ñ@Ô@��à#Ô-°Ò1Ð1Ø3>Ô3CÀaÒ3GÐ3G˜K¨Ô/Ð/È[�	Ý<¸UÀIÑNÔN��à#Ô+¨aÒ/Ð/Ø/:Ô/?À!Ò/CÐ/C˜+ dÔ+Ð+È�Ý2°5¸'ÑBÔB�Øˆr2   Úis_main_processÚboolc                óN   — |}t          ¦   «         r|ot          ¦   «         dk    }|S )z7Return whether this rank should write checkpoint files.r   )r   r   )rC   rˆ   Úsave_on_this_ranks      r/   Úshould_save_on_this_rankz)DistributedMixin.should_save_on_this_rankÒ   s5   € à+ÐÝ,Ñ.Ô.ð 	YØ 1Ð XÕ6QÑ6SÔ6SÐWXÒ6XÐØ Ð r2   FT)Úpush_to_hubr‹   Úrepo_idÚfiles_timestampsÚcommit_messageÚtokenÚ	create_prÚsave_directoryústr | os.PathLiker�   r‹   rŽ   ú
str | Noner�   údict | Noner�   r‘   ústr | bool | Noner’   c               ó¸  — t          d¦  «        st          d¦  «        ‚t          |¦  «        st          d¦  «        ‚t	          |dd¦  «        €t          d¦  «        ‚t          ||¦  «         |ri|rit          || j        |¬¦  «        }
|
                     t          j
                             |d¦  «        ¦  «         |                      ||||||	¬	¦  «         dS dS dS )
zBSave an FSDP-wrapped model via DCP and optionally push to the Hub.z2.7zFsave_pretrained(..., distributed_checkpoint=True) requires torch>=2.7.z\save_pretrained(..., distributed_checkpoint=True) is only supported for FSDP-wrapped models.r�   Nz�save_pretrained(..., distributed_checkpoint=True) requires the model to have been initialized with a distributed_config (_device_mesh is None).)r‘   z	README.md)r�   r‘   r’   )r   ÚOSErrorr   rM   r8   r   r   Ú
model_tagsÚsaveÚosÚpathÚjoinÚ_upload_modified_files)rC   Úmodel_to_saver“   r�   r‹   rŽ   r�   r�   r‘   r’   Ú
model_cards              r/   Úsave_distributed_checkpointz,DistributedMixin.save_distributed_checkpointÙ   s  € õ )¨Ñ/Ô/ð 	dÝÐbÑcÔcÐcÝ% mÑ4Ô4ð 	ÝØnñô ð õ �= .°$Ñ7Ô7Ð?ÝðPñô ð õ 	*¨-¸ÑHÔHÐHàð 
	Ð,ð 
	Ý2°7¸D¼OÐSXÐYÑYÔYˆJØ�OŠO�BœGŸLšL¨¸ÑEÔEÑFÔFÐFØ×'Ò'ØØØ Ø-ØØ#ð (ñ ô ð ð ð ð
	ð 
	ð 
	ð 
	r2   )r‹   Ú
state_dictr7   c               óà   — |€|S |j         dk    r't          || j        | j        |j         ¦  «        }|si }|S |j        dk    r,t          ¦   «         st          d¦  «        ‚t          |¦  «        S |S )zNGather TP- or FSDP-sharded weights to full CPU tensors for checkpoint writing.Nr	   z‰Saving an FSDP-wrapped model requires torch.distributed to be initialized. Call save_pretrained from every rank after init_process_group.)rr   r   r   r�   rt   r   rM   r   )rC   r    r£   rJ   r‹   s        r/   Ú"gather_sharded_state_dict_for_savez3DistributedMixin.gather_sharded_state_dict_for_save   sž   € ð Ð%ØÐàÔ%¨Ò)Ð)Ý3Ø˜DœM¨4Ô+<Ð>PÔ>Xñô ˆJð %ð  Ø�
ØÐàÔ'¨!Ò+Ð+Ý4Ñ6Ô6ð Ý ðUñô ð õ *¨-Ñ8Ô8Ð8àÐr2   c                óZ   — |€dS |j         dk    s|j        dk    rt          ¦   «          dS dS )zQBarrier so non-writer ranks wait for rank 0 to finish gathered checkpoint writes.Nr	   )rr   rt   r   )rC   rJ   s     r/   Ú&barrier_after_gathered_checkpoint_savez7DistributedMixin.barrier_after_gathered_checkpoint_save  sD   € àÐ%ØˆFØÔ%¨Ò)Ð)Ð-?Ô-IÈAÒ-MÐ-MÝ Ñ"Ô"Ð"Ð"Ð"ð .NÐ-Mr2   )r#   r$   )r#   rH   )r#   rU   )rF   r   )rF   r    )rJ   rm   r#   rn   )rz   r{   rJ   r|   )rˆ   r‰   r#   r‰   )r“   r”   r�   r‰   r‹   r‰   rŽ   r•   r�   r–   r�   r•   r‘   r—   r’   r‰   r#   r$   )r£   r7   rJ   r|   r‹   r‰   r#   r7   )rJ   r|   r#   r$   )rO   Ú
__module__Ú__qualname__Ú__doc__r�   r   Ú__annotations__r   r‚   r!   r"   rG   ÚpropertyrQ   rT   rW   ÚsetterÚclassmethodry   r‡   rŒ   r¢   r¥   r§   r)   r2   r/   r   r   4   sú  € € € € € € ØPÐPà€LØ&*€HÐ*Ð*Ð*Ñ*Ø&*€HÐ*Ð*Ð*Ñ*Ø€HØ26€HÐ6Ð6Ð6Ñ6Ø(,€JÐ,Ð,Ð,Ñ,ð[ð [ð [ð [ð0 ð
ð 
ð 
ñ „Xð
ð ðð ð ñ „Xðð ðð ð ñ „Xðð „^ðð ð ñ „^ðð2 „^ðð ð ñ „^ðð ð
 Øð$;ð $;ð $;ð $;ð $;ñ „[ð$;ðL ðð ð ñ „[ðð6!ð !ð !ð !ð "Ø"&Ø"Ø(,Ø%)Ø#'Øð%ð %ð %ð %ð %ð %ðZ #'ðð ð ð ð ð ð<#ð #ð #ð #ð #ð #r2   r   )'Ú
__future__r   rœ   ra   rc   Útypingr   Úutilsr   r   Ú	utils.hubr   Úconfiguration_utilsr
   r   r   r   Úpipeline_parallelr   Útensor_parallelr   r   r   r   r   r   r   r   r   r   r   r   r   Ú
get_loggerrO   ÚloggerÚtorch.nnÚnnr   r)   r2   r/   ú<module>rº      sÂ  ðð #Ð "Ð "Ð "Ð "Ð "à 	€	€	€	Ø 	€	€	€	Ø €€€Ø  Ð  Ð  Ð  Ð  Ð  à 6Ð 6Ð 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø 2Ð 2Ð 2Ð 2Ð 2Ð 2Ø NÐ NÐ NÐ NÐ NÐ NÐ NÐ NØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð
ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð 
ˆÔ	˜HÑ	%Ô	%€ð ð ØÐÐÐÐÐðo#ð o#ð o#ð o#ð o#ñ o#ô o#ð o#ð o#ð o#r2   