§
    “©ºjan  ã                   ó¢  — d dl Z d dlZd dlZd dlmZ d dlmZ d dlmZm	Z	m
Z
 d dlZd dlmZ d dlZd dlmZ d dlmZ d dlmZ d d	lmZmZ d d
lmZmZ d dlmZ d dlmZ d dl m!Z!m"Z" d dl#m$Z$ d dl%m&Z& d dl'm(Z( d dl)m*Z*m+Z+ d dl,m-Z-m.Z. d dl/m0Z0m1Z1 d dl2m3Z3 e	e
eef                  Z4e G d„ d¦  «        ¦   «         Z5 G d„ dej6        ¦  «        Z7dS )é    N)ÚOrderedDict)Ú	dataclass)ÚAnyÚOptionalÚUnion)Úoverride)Ú	_Stateful)ÚPossibleUserWarning)Úloops)Ú_DataFetcherÚ_DataLoaderIterDataFetcher)Ú_AutomaticOptimizationÚ_ManualOptimization)Ú_OUTPUTS_TYPE)Ú_BatchProgressÚ_SchedulerProgress)Ú_is_max_limit_reached)Úcall)Ú_ResultCollection)ÚRunningStageÚ	TrainerFn)ÚMisconfigurationExceptionÚSIGTERMException)ÚWarningCacheÚrank_zero_warn)Úis_param_in_hook_signaturec                   ó   — e Zd ZdZdZdZdS )ÚRestartStageÚnoneÚrestarted_on_train_batch_endÚrestarted_on_lastN)Ú__name__Ú
__module__Ú__qualname__ÚNONEÚRESTARTED_ON_TRAIN_BATCH_ENDÚRESTARTED_ON_LAST© ó    ú{/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/pytorch_lightning/loops/training_epoch_loop.pyr   r   ,   s!   € € € € € à€DØ#AÐ Ø+ÐÐÐr)   r   c            	       ó~  ‡ — e Zd ZdZd1dddee         deddfˆ fd	„Zedefd
„¦   «         Zedefd„¦   «         Z	edefd„¦   «         Z
edefd„¦   «         Zedefd„¦   «         Zedefd„¦   «         Zdeddfd„Zedefd„¦   «         Zedefd„¦   «         Zd2d„Zd2d„Zd2d„Zdeddfd„Zd2d„Zd2d„Zd2d„Zdeddfd„Zdeddfd„Zd2d„Zedefˆ fd„¦   «         Zededdfd „¦   «         Z defd!„Z!defd"„Z"defd#„Z#d$e$d%eddfd&„Z%d$e$d%eddfd'„Z&d(e$dee'         fd)„Z(defd*„Z)dedefd+„Z*d2d,„Z+d-e,d.e'd/ede,fd0„Z-ˆ xZ.S )3Ú_TrainingEpochLoopaô  Iterates over all batches in the dataloader (one epoch) that the user returns in their
    :meth:`~pytorch_lightning.core.LightningModule.train_dataloader` method.

    Its main responsibilities are calling the ``*_epoch_{start,end}`` hooks, accumulating outputs if the user request
    them in one of these hooks, and running validation at the requested interval.

    The validation is carried out by yet another loop,
    :class:`~pytorch_lightning.loops._EvaluationLoop`.

    In the ``run()`` method, the training epoch loop could in theory simply call the
    ``LightningModule.training_step`` already and perform the optimization.
    However, Lightning has built-in support for automatic optimization with multiple optimizers.
    For this reason there are actually two more loops nested under
    :class:`~pytorch_lightning.loops._TrainingEpochLoop`.

    Args:
        min_steps: The minimum number of steps (batches) to process
        max_steps: The maximum number of steps (batches) to process

    NéÿÿÿÿÚtrainerz
pl.TrainerÚ	min_stepsÚ	max_stepsÚreturnc                 ó&  •— t          ¦   «                              |¦  «         |dk     rt          d|› d�¦  «        ‚|| _        || _        t          ¦   «         | _        t          ¦   «         | _        t          |¦  «        | _
        t          |¦  «        | _        t          j        |t          j        t"          j        dd¬¦  «        | _        t)          d¬¦  «        | _        t-          ¦   «         | _        d| _        t2          j        | _        d| _        d S )	Nr-   zQ`max_steps` must be a non-negative integer or -1 (infinite steps). You passed in ú.F)ÚverboseÚinference_modeT)Útrainingr   )ÚsuperÚ__init__r   r/   r0   r   Úbatch_progressr   Úscheduler_progressr   Úautomatic_optimizationr   Úmanual_optimizationr   Ú_EvaluationLoopr   ÚFITTINGr   Ú
VALIDATINGÚval_loopr   Ú_resultsr   Ú_warning_cacheÚ_batches_that_steppedr   r%   Ú_restart_stageÚ_skip_next_val)Úselfr.   r/   r0   Ú	__class__s       €r*   r8   z_TrainingEpochLoop.__init__I   sý   ø€ Ý‰Œ×Ò˜Ñ!Ô!Ð!Ø�rŠ>ˆ>Ý+ØpÐdmÐpÐpÐpñô ð ð #ˆŒØ"ˆŒå,Ñ.Ô.ˆÔÝ"4Ñ"6Ô"6ˆÔå&<¸WÑ&EÔ&EˆÔ#Ý#6°wÑ#?Ô#?ˆÔ åÔ-Ø•YÔ&­Ô(?ÈÐ_dð
ñ 
ô 
ˆŒõ *°4Ð8Ñ8Ô8ˆŒÝ*™nœnˆÔØ*+ˆÔ"Ý*Ô/ˆÔØ#ˆÔÐÐr)   c                 ó*   — | j         j        j        dz
  S )z/Returns the current batch index (across epochs)é   )r9   ÚtotalÚready©rF   s    r*   Útotal_batch_idxz"_TrainingEpochLoop.total_batch_idxb   s   € ð
 Ô"Ô(Ô.°Ñ2Ð2r)   c                 ó*   — | j         j        j        dz
  S )z3Returns the current batch index (within this epoch)rI   )r9   ÚcurrentrK   rL   s    r*   Ú	batch_idxz_TrainingEpochLoop.batch_idxi   s   € ð
 Ô"Ô*Ô0°1Ñ4Ð4r)   c                 óz   — | j         j        }|�|j        r| j        j        j        S | j        j        j        j        S ©N)	r.   Úlightning_moduler;   Úoptim_progressÚoptimizer_stepsr<   Úoptim_step_progressrJ   Ú	completed)rF   rS   s     r*   Úglobal_stepz_TrainingEpochLoop.global_stepp   s=   € àœ<Ô8ÐØÐ#Ð'7Ô'NÐ#ØÔ.Ô=ÔMÐMØÔ'Ô;ÔAÔKÐKr)   c                 ób   — t          | j        | j        ¦  «        }|p|                      ¦   «         S rR   )r   rX   r0   Ú_num_ready_batches_reached)rF   Úmax_steps_reacheds     r*   Ú_is_training_donez$_TrainingEpochLoop._is_training_donew   s.   € å1°$Ô2BÀDÄNÑSÔSÐØ ÐE D×$CÒ$CÑ$EÔ$EÐEr)   c                 ó*   — | j          p| j        j        S rR   )Ú
restartingr@   Ú_has_runrL   s    r*   Ú_is_validation_donez&_TrainingEpochLoop._is_validation_done|   s   € ð ”?Ð"Ð< d¤mÔ&<Ð<r)   c                 óÖ   — | j         r	| j        rdS | j        j        rL| j        j        j        }| j        j        j        }|s&| j                             d|›d| j	        ›d�¦  «         |S dS )z!Evaluates when to leave the loop.Tz:Trainer was signaled to stop but the required `min_epochs=z` or `min_steps=z-` has not been met. Training will continue...F)
r\   r`   r.   Úshould_stopÚfit_loopÚ
min_epochsÚ_can_stop_earlyrB   Úinfor/   )rF   rd   Úcan_stop_earlys      r*   Údonez_TrainingEpochLoop.done�   s¡   € ð Ô!ð 	 dÔ&>ð 	Ø�4àŒ<Ô#ð 		"àœÔ.Ô9ˆJØ!œ\Ô2ÔBˆNØ!ð ØÔ#×(Ò(ðcÐQ[ð cð cØ#'¤>ðcð cð cñô ð ð "Ð!àˆur)   Údata_fetcherc                 óx  — |                       ¦   «          |                      |¦  «         | j        s‰	 |                      |¦  «         |                      |¦  «         n%# t
          $ r Y |                      ¦   «          d S w xY w	 |                      ¦   «          n# |                      ¦   «          w xY w| j        ¯‡d S d S rR   )ÚresetÚon_run_startrh   ÚadvanceÚon_advance_endÚStopIterationÚon_iteration_done©rF   ri   s     r*   Úrunz_TrainingEpochLoop.run”   så   € Ø�
Š
‰ŒˆØ×Ò˜,Ñ'Ô'Ð'Ø”)ð 	)ð)Ø—’˜\Ñ*Ô*Ð*Ø×#Ò# LÑ1Ô1Ð1Ð1øÝ ð ð ð Øà×&Ò&Ñ(Ô(Ð(Ð(Ð(ðøøøð 2ð ×&Ò&Ñ(Ô(Ð(Ð(ø�×&Ò&Ñ(Ô(Ð(Ð(øøøð ”)ð 	)ð 	)ð 	)ð 	)ð 	)s)   ²*A ÁB Á
A?Á'B Á>A?Á?B ÂB.c                 ó,   — | j         t          j        k    S rR   )rD   r   r&   rL   s    r*   r    z/_TrainingEpochLoop.restarted_on_train_batch_end    s   € àÔ"¥lÔ&OÒOÐOr)   c                 ó,   — | j         t          j        k    S rR   )rD   r   r'   rL   s    r*   r!   z$_TrainingEpochLoop.restarted_on_last¤   s   € àÔ"¥lÔ&DÒDÐDr)   c                 ót  — | j         r�| j        j        j        | j        j        j        k    r]| j        j        j        | j        j        j        k    r9| j        j        j        | j        j        j        dz
  k    rt          j        | _	        n–| j         r~| j        j        j        | j        j        j        k    rZ| j        j        j        | j        j        j        k    r6| j        j        j        | j        j        j        k    rt          j
        | _	        nt          j        | _	        | j                             ¦   «          d S )NrI   )r^   r9   rJ   ÚstartedrK   Ú	processedrW   r   r&   rD   r'   r%   r@   Úupdate_restart_stagerL   s    r*   rx   z'_TrainingEpochLoop.update_restart_stage¨   s  € àŒOð	4àÔ#Ô)Ô1°TÔ5HÔ5NÔ5TÒTÐTØÔ#Ô)Ô3°tÔ7JÔ7PÔ7XÒXÐXØÔ#Ô)Ô3°tÔ7JÔ7PÔ7ZÐ]^Ñ7^Ò^Ð^å".Ô"KˆDÔÐàŒOð	4àÔ#Ô)Ô1°TÔ5HÔ5NÔ5TÒTÐTØÔ#Ô)Ô3°tÔ7JÔ7PÔ7XÒXÐXØÔ#Ô)Ô3°tÔ7JÔ7PÔ7ZÒZÐZå".Ô"@ˆDÔÐå".Ô"3ˆDÔàŒ×*Ò*Ñ,Ô,Ð,Ð,Ð,r)   c                 ó(   — t           j        | _        d S rR   )r   r%   rD   rL   s    r*   Úreset_restart_stagez&_TrainingEpochLoop.reset_restart_stage¼   s   € Ý*Ô/ˆÔÐÐr)   c                 ó¨  — | j         r2|                      ¦   «         s| j        s| j        s| xj        dz  c_        | j        r«| j                             ¦   «          | j        j        j        | j	        j
        k    rs| j                             ¦   «          | j                             ¦   «          | j        j                             ¦   «          | j        j        j                             ¦   «          | j         rì| j                             ¦   «          | j                             ¦   «          | j        j                             ¦   «          | j	        }|j
        t'          d¦  «        k    rwt)          j        |j
        |j        z  ¦  «        }|j        j        }|€J ‚t3          d„ |j        D ¦   «         ¦  «        }| j        |z  dk    r|st9          dt:          ¬¦  «         dS dS dS dS | j                             ¦   «          | j                             ¦   «          | j        j                             ¦   «          | j        j        j                             ¦   «          dS )z4Resets the internal state of the loop for a new run.rI   ÚinfNc              3   ó@   K  — | ]}t          |t          ¦  «        V — Œd S rR   )Ú
isinstancer	   )Ú.0Úloaders     r*   ú	<genexpr>z+_TrainingEpochLoop.reset.<locals>.<genexpr>Þ   s,   è è € Ð)gÐ)gÈF­*°V½YÑ*GÔ*GÐ)gÐ)gÐ)gÐ)gÐ)gÐ)gr)   r   a4  You're resuming from a checkpoint that ended before the epoch ended and your dataloader is not resumable. This can cause unreliable results if further training is done. Consider using an end-of-epoch checkpoint or make your dataloader resumable by implementing the `state_dict` / `load_state_dict` interface.©Úcategory)r^   Ú_should_accumulater    r!   rC   r9   Úincrement_completedrO   rW   r.   Únum_training_batchesÚreset_on_runr:   r;   rT   r@   rJ   rk   Úreset_on_restartÚfloatÚmathÚceilÚaccumulate_grad_batchesrc   Ú_combined_loaderÚallÚ	flattenedrX   r   r
   )rF   r.   Úexpected_stepsr€   Úis_resumable_loaders        r*   rk   z_TrainingEpochLoop.reset¿   sa  € ð ŒOð	,à×+Ò+Ñ-Ô-ð	,ð Ô2ð	,ð ;?Ô:Pð	,ð Ð&Ô&¨!Ñ+Ð&Ô&àÔ,ð 	;ØÔ×3Ò3Ñ5Ô5Ð5àÔ"Ô*Ô4¸¼Ô8YÒYÐYØÔ#×0Ò0Ñ2Ô2Ð2ØÔ'×4Ò4Ñ6Ô6Ð6ØÔ+Ô:×GÒGÑIÔIÐIØ”Ô,Ô2×8Ò8Ñ:Ô:Ð:àŒ?ð 	7ØÔ×0Ò0Ñ2Ô2Ð2ØÔ#×4Ò4Ñ6Ô6Ð6ØÔ'Ô6×GÒGÑIÔIÐIà”lˆGØÔ+­u°U©|¬|Ò;Ð;Ý!%¤¨7Ô+GÈ'ÔJiÑ+iÑ!jÔ!j�Ø Ô)Ô:�ØÐ)Ð)Ð)Ý&)Ð)gÐ)gÐV\ÔVfÐ)gÑ)gÔ)gÑ&gÔ&gÐ#ØÔ# nÑ4¸Ò9Ð9ÐBUÐ9Ý"ðKõ "5ðñ ô ð ð ð ð <Ð;ð
 :Ð9Ð9Ð9ð Ô×,Ò,Ñ.Ô.Ð.ØÔ#×0Ò0Ñ2Ô2Ð2ØÔ'Ô6×CÒCÑEÔEÐEð ŒMÔ(Ô.×4Ò4Ñ6Ô6Ð6Ð6Ð6r)   c                 óâ   — | j         j        dk    r'| j         j        j        s| j        st          |¦  «         |xj        | j        j        j	        z  c_        | j
        |_        | j        |_        d S )Nr   )r.   Úcurrent_epochrc   Úis_resumingr^   ÚiterÚfetchedr9   rO   rK   Ú_on_before_fetchÚ_start_profilerÚ_on_after_fetchÚ_stop_profilerrq   s     r*   rl   z_TrainingEpochLoop.on_run_startï   ss   € ð Œ<Ô%¨Ò)Ð)°$´,Ô2GÔ2SÐ)Ð\`Ô\kÐ)Ý�ÑÔÐð 	ÐÔ Ô 3Ô ;Ô AÑAÐÔØ'+Ô'<ˆÔ$Ø&*Ô&:ˆÔ#Ð#Ð#r)   c                 ó`   — | j         j                             d| j        j        › d�¦  «         d S ©Nú[z].train_dataloader_next)r.   ÚprofilerÚstartrG   r"   rL   s    r*   r—   z#_TrainingEpochLoop._on_before_fetchý   s2   € ØŒÔ×#Ò#Ð$X¨¬Ô(?Ð$XÐ$XÐ$XÑYÔYÐYÐYÐYr)   c                 ó`   — | j         j                             d| j        j        › d�¦  «         d S rœ   )r.   rž   ÚstoprG   r"   rL   s    r*   r™   z"_TrainingEpochLoop._on_after_fetch   s2   € ØŒÔ×"Ò"Ð#W t¤~Ô'>Ð#WÐ#WÐ#WÑXÔXÐXÐXÐXr)   c                 ó  — 	 t          j        t          | j        dd¦  «        rdndg| j        j        j        ¬¦  «        }t           j                             |d¬¦  «         n6# t          $ r) t          j        dg| j        j        j        ¬¦  «        }Y nw xY w| 	                    ¦   «         dk    r]t          j        t          ¦  «        5  t           j                             ¦   «          d d d ¦  «         n# 1 swxY w Y   t          ¦   «         ‚d S )NÚreceived_sigtermFrI   r   )Údevice)Úsrc)ÚtorchÚtensorÚgetattrr.   ÚstrategyÚroot_deviceÚdistributedÚ	broadcastÚ	ExceptionÚitemÚ
contextlibÚsuppressÚbarrierr   )rF   Úsigterm_tensors     r*   Ú_broadcast_sigterm_tensorz,_TrainingEpochLoop._broadcast_sigterm_tensor  sQ  € ð	YÝ"œ\Ý˜dœlÐ,>ÀÑFÔFÐM��ÈAÐNØ”|Ô,Ô8ðñ ô ˆNõ Ô×'Ò'¨¸AÐ'Ñ>Ô>Ð>Ð>øÝð 	Yð 	Yð 	YÝ"œ\¨1¨#°d´lÔ6KÔ6WÐXÑXÔXˆNˆNˆNð	Yøøøð ×ÒÑ Ô  AÒ%Ð%ÝÔ$¥YÑ/Ô/ð ,ð ,ÝÔ!×)Ò)Ñ+Ô+Ð+ð,ð ,ð ,ñ ,ô ,ð ,ð ,ð ,ð ,ð ,ð ,øøøð ,ð ,ð ,ð ,å"Ñ$Ô$Ð$ð &Ð%s$   ‚AA" Á"0BÂBÃ	C4Ã4C8Ã;C8c                 ó  — | j         rL|                      |¦  «        r7| j        j        rdS | j        r	d| _        dS | j                             ¦   «          d| j        _         t          j         	                    ¦   «         rBt          j         
                    ¦   «         r$| j        j        dk    r|                      ¦   «          t          |t          ¦  «        x}rt!          |¦  «        }|j        }|j        }nd}t!          |¦  «        \  }}}| j        dz   }|j        | j        _        | j        }|sN|j                             |¦  «        }|j                             |d¬¦  «        }t7          j        |d|d¬¦  «        }| j                             ¦   «          |j                             |¦  «         d}	d}
|€|s| j          !                    d¦  «         �n8t7          j"        |d	||¦  «         t7          j#        |d	||¦  «        }t7          j        |d	||¦  «         |d
k    }
|
r| j         $                    dd¬¦  «         nÎ| j         %                    ¦   «          |s#|  &                    tO          ¦   «         ||¦  «        ntO          |¬¦  «        }|j(         )                    d¦  «        5  |j        j*        r(| j*         +                    |j,        d         ||¦  «        }	n| j-         +                    |¦  «        }	ddd¦  «         n# 1 swxY w Y   | j         .                    ¦   «          |  /                    dd¬¦  «         |  0                    ¦   «         r|  /                    dd¬¦  «         |
rtb          ‚|r|j        }|j        }|j        | j        _        t7          j"        |d|	||¦  «         t7          j#        |d|	||¦  «         |j         2                    ¦   «          | j         3                    ¦   «          |j         4                    ¦   «          dS )z‡Runs a single training batch.

        Raises:
            StopIteration: When the epoch is canceled by the user returning -1

        NTFrI   r   )Údataloader_idxÚbatch_to_devicezMtrain_dataloader yielded None. If this was on purpose, ignore this warning...Úon_train_batch_startr-   )Úis_last_batch)ÚanyÚrun_training_batchÚstep©Úupdate_plateau_schedulersÚepochÚon_train_batch_end)5r^   Ú_should_check_val_fxr@   Úrestarted_mid_evaluationr!   rE   Ú$increment_progress_to_evaluation_endr¦   r«   Úis_availableÚis_initializedr.   Ú
world_sizer³   r~   r   ÚnextÚ_batchÚ
_batch_idxrP   rh   r9   r¸   Úprecision_pluginÚconvert_inputrS   Ú_on_before_batch_transferr   Ú_call_strategy_hookÚincrement_readyÚ_logger_connectorÚon_batch_startrB   ÚwarnÚ_call_callback_hooksÚ_call_lightning_module_hookÚincrement_byÚincrement_startedÚ_build_kwargsr   rž   Úprofiler;   rr   Ú
optimizersr<   Úincrement_processedÚupdate_lr_schedulersrZ   ro   Úon_batch_endr…   Úupdate_train_step_metrics)rF   ri   Úusing_dataloader_iterÚdataloader_iterÚbatchrP   Ú_Ú__r.   Úbatch_outputÚshould_skip_rest_of_epochÚresponseÚkwargss                r*   rm   z_TrainingEpochLoop.advance  s”  € ð Œ?ð 	A˜t×8Ò8¸ÑFÔFð 	AØŒ}Ô5ð à�àÔ%ð à&*�Ô#Ø�ð ŒM×>Ò>Ñ@Ô@Ð@ð $)ˆŒÔ õ Ô×)Ò)Ñ+Ô+ð 	-µÔ0A×0PÒ0PÑ0RÔ0Rð 	-ÐW[ÔWcÔWnÐqrÒWrÐWrØ×*Ò*Ñ,Ô,Ð,õ %/¨|Õ=WÑ$XÔ$XÐXÐ ð 
	+Ý" <Ñ0Ô0ˆOà Ô'ˆEØ$Ô/ˆIˆIà"ˆOÝ Ñ-Ô-‰LˆE�1�bð œ¨Ñ*ˆIà,8Ô,=ˆÔÔ)à”,ˆØ$ð 	bØÔ,×:Ò:¸5ÑAÔAˆEØÔ,×FÒFÀuÐ]^ÐFÑ_Ô_ˆEÝÔ,¨WÐ6GÈÐ_`ÐaÑaÔaˆEàÔ×+Ò+Ñ-Ô-Ð-ØÔ!×0Ò0°Ñ7Ô7Ð7à,0ˆØ$)Ð!àˆ=Ð!6ˆ=ØÔ×$Ò$Ð%tÑuÔuÐuÑuõ Ô% gÐ/EÀuÈiÑXÔXÐXÝÔ7¸ÐAWÐY^Ð`iÑjÔjˆHÝÔ$ WÐ.DÀeÈYÑWÔWÐWØ(0°BªÐ%à(ð LØÔ#×0Ò0°À$Ð0ÑGÔGÐGÐGàÔ#×5Ò5Ñ7Ô7Ð7ð 1ð:�D×&Ò&¥{¡}¤}°e¸YÑGÔGÐGå$¨Ð9Ñ9Ô9ð ð
 Ô%×-Ò-Ð.BÑCÔCð Lð LØÔ/ÔFð Là'+Ô'B×'FÒ'FÀwÔGYÐZ[ÔG\Ð^gÐioÑ'pÔ'p˜˜à'+Ô'?×'CÒ'CÀFÑ'KÔ'K˜ðLð Lð Lñ Lô Lð Lð Lð Lð Lð Lð Løøøð Lð Lð Lð Lð 	Ô×/Ò/Ñ1Ô1Ð1ð 	×!Ò! &ÀEÐ!ÑJÔJÐJØ×*Ò*Ñ,Ô,ð 	PØ×%Ò% gÈÐ%ÑOÔOÐOà$ð 	 åÐà ð 	Bà Ô'ˆEØ$Ô/ˆIà0<Ô0AˆDÔÔ-åÔ! 'Ð+?ÀÈuÐV_Ñ`Ô`Ð`ÝÔ(¨Ð2FÈÐV[Ð]fÑgÔgÐgØÔ!×.Ò.Ñ0Ô0Ð0àÔ×/Ò/Ñ1Ô1Ð1ð
 	Ô!×;Ò;Ñ=Ô=Ð=Ð=Ð=s   Ê%AL Ì LÌLc                 ó,  — |                       |¦  «        }| j        r	d}d| _        |r�d| j        _        | j        j        j        }|                      ¦   «         st          j        | j        d¦  «         | j	         
                    ¦   «          d| j        _        || j        j        _        |                      dd¬¦  «         |                      ¦   «         s| xj        dz  c_        |                      ¦   «          | j        s| j        j        r	t"          ‚d S d S )NFTÚon_validation_model_zero_gradr»   r¼   rI   )rÀ   rE   r.   Ú
validatingrÎ   Ú_first_loop_iterr„   r   rÒ   r@   rr   r6   rÙ   rC   Ú _save_loggers_on_train_batch_endr\   r£   r   )rF   ri   Úshould_check_valÚfirst_loop_iters       r*   rn   z!_TrainingEpochLoop.on_advance_end‚  s/  € ð  ×4Ò4°\ÑBÔBÐàÔð 	(Ø$ÐØ"'ˆDÔàð 	Nà&*ˆDŒLÔ#à"œlÔ<ÔMˆOà×*Ò*Ñ,Ô,ð `åÔ0°´Ð?^Ñ_Ô_Ð_àŒM×ÒÑÔÐØ$(ˆDŒLÔ!Ø>MˆDŒLÔ*Ô;ð 	×!Ò! &ÀDÐ!ÑIÔIÐIà×&Ò&Ñ(Ô(ð 	,àÐ&Ô&¨!Ñ+Ð&Ô&à×-Ò-Ñ/Ô/Ð/ð Ô%ð 	#¨$¬,Ô*Gð 	#Ý"Ð"ð	#ð 	#ð 	#ð 	#r)   c                 ój   — | j                              ¦   «          | j                             ¦   «          d S rR   )rA   Úcpur@   ÚteardownrL   s    r*   rî   z_TrainingEpochLoop.teardown¨  s0   € ØŒ×ÒÑÔÐØŒ×ÒÑ Ô Ð Ð Ð r)   c                 ó\   •— t          ¦   «                              ¦   «         }| j        |d<   |S )NrC   )r7   Úon_save_checkpointrC   )rF   Ú
state_dictrG   s     €r*   rð   z%_TrainingEpochLoop.on_save_checkpoint¬  s,   ø€ å‘W”W×/Ò/Ñ1Ô1ˆ
Ø.2Ô.Hˆ
Ð*Ñ+ØÐr)   rñ   c                 ó<   — |                      dd¦  «        | _        d S )NrC   r   )ÚgetrC   )rF   rñ   s     r*   Úon_load_checkpointz%_TrainingEpochLoop.on_load_checkpoint²  s   € à%/§^¢^Ð4KÈQÑ%OÔ%OˆÔ"Ð"Ð"r)   c                 óF   — | j         j        j        | j        j        z  dk    S )zKDetermine if accumulation will be finished by the end of the current batch.r   )r9   rO   rK   r.   rŒ   rL   s    r*   Ú_accumulated_batches_reachedz/_TrainingEpochLoop._accumulated_batches_reached¶  s!   € àÔ"Ô*Ô0°4´<Ô3WÑWÐ[\Ò\Ð\r)   c                 ó\   — | j         j        j        | j        j        k    }|p| j         j        S )zJChecks if we are in the last batch or if there are more batches to follow.)r9   rO   rK   r.   r†   r¸   )rF   Úepoch_finished_on_readys     r*   rZ   z-_TrainingEpochLoop._num_ready_batches_reachedº  s,   € à"&Ô"5Ô"=Ô"CÀtÄ|ÔGhÒ"hÐØ&ÐK¨$Ô*=Ô*KÐKr)   c                 ó„   — |                       ¦   «         }|                      ¦   «         }| j        j        j        p| }| o|S )ziChecks if the optimizer step should be performed or gradients should be accumulated for the current step.)rö   rZ   r.   r©   Úhandles_gradient_accumulation)rF   Úaccumulation_doneÚis_final_batchÚ#strategy_accumulates_on_final_batchs       r*   r„   z%_TrainingEpochLoop._should_accumulate¿  sK   € à ×=Ò=Ñ?Ô?Ðà×8Ò8Ñ:Ô:ˆà.2¬lÔ.CÔ.aÐ.wÐiwÐewÐ+Ø$Ð$ÐLÐ)LÐLr)   Úintervalr½   c                 ól   — |dk    r|                       ¦   «         rdS |                      ||¬¦  «         dS )z6Updates the lr schedulers based on the given interval.r»   N)rþ   r½   )r„   Ú_update_learning_rates)rF   rþ   r½   s      r*   rÙ   z'_TrainingEpochLoop.update_lr_schedulersÈ  sC   € à�vÒÐ $×"9Ò"9Ñ";Ô";ÐØˆFØ×#Ò#¨XÐQjÐ#ÑkÔkÐkÐkÐkr)   c                 óZ  — | j         }|j        r|j        j        sdS |j        D �]}||j        z  rŒ|dk    r| j        n|j        }|dz  }|j        |k    rÓ||j        z  dk    rÅd}|j        rn|j	        }|€J ‚|  
                    |¦  «        }|€L|j        r*t          |j        ¦  «        }t          d|› d|› d�¦  «        ‚t          d|› d�t           ¬	¦  «         Œ·| j                             ¦   «          t'          j        |d
|j        |¦  «         | j                             ¦   «          �ŒdS )aî  Update learning rates.

        Args:
            interval: either 'epoch' or 'step'.
            update_plateau_schedulers: control whether ``ReduceLROnPlateau`` or non-plateau schedulers get updated.
                This is used so non-plateau schedulers can be updated before running validation. Checkpoints are
                commonly saved during validation, however, on-plateau schedulers might monitor a validation metric
                so they have to be updated separately.

        Nr»   rI   r   z(ReduceLROnPlateau conditioned on metric z0 which is not available. Available metrics are: z?. Condition can be set using `monitor` key in lr scheduler dictzT which is not available but strict is set to `False`. Skipping learning rate update.r‚   Úlr_scheduler_step)r.   Úlr_scheduler_configsrS   r;   Úreduce_on_plateaurP   r“   rþ   Ú	frequencyÚmonitorÚ_get_monitor_valueÚstrictÚlistÚcallback_metricsr   r   ÚRuntimeWarningr:   rÍ   r   rÒ   Ú	schedulerr…   )	rF   rþ   r½   r.   ÚconfigÚcurrent_idxÚmonitor_valÚmonitor_keyÚavail_metricss	            r*   r   z)_TrainingEpochLoop._update_learning_ratesÎ  s¹  € ð ”,ˆàÔ+ð 	°7Ô3KÔ3bð 	ØˆFàÔ2ð '	>ñ '	>ˆFØ(¨6Ô+CÑCð Øà,4¸Ò,>Ð,>˜$œ.˜.ÀGÔDYˆKØ˜1ÑˆKð Œ (Ò*Ð*¨{¸VÔ=MÑ/MÐQRÒ/RÐ/RØ"�ØÔ+ð !Ø"(¤.�KØ&Ð2Ð2Ð2Ø"&×"9Ò"9¸+Ñ"FÔ"F�KØ"Ð*Ø!œ=ð Ý,0°Ô1IÑ,JÔ,J˜MÝ";ð!aÈ;ð !að !aØS`ð!að !að !añ#ô #ð õ
 'ð>À{ð >ð >ð >õ &4ð	ñ ô ð ð !àÔ'×7Ò7Ñ9Ô9Ð9õ Ô0ØØ'ØÔ$Øñ	ô ð ð Ô'×;Ò;Ñ=Ô=Ð=ùðO'	>ð '	>r)   Úkeyc                 ó@   — | j         j                             |¦  «        S rR   )r.   r
  ró   )rF   r  s     r*   r  z%_TrainingEpochLoop._get_monitor_value  s   € àŒ|Ô,×0Ò0°Ñ5Ô5Ð5r)   c                 óv   — | j         j        o-| j         j        d u p| j         j        dz   | j         j        z  dk    S )NrI   r   )r.   Úenable_validationÚcheck_val_every_n_epochr“   rL   s    r*   Ú_should_check_val_epochz*_TrainingEpochLoop._should_check_val_epoch  sD   € ØŒ|Ô-ð 
ØŒLÔ0°DÐ8ð \Ø”Ô*¨QÑ.°$´,Ô2VÑVÐZ[Ò[ð	
r)   c                 ó”  — |                       ¦   «         sdS | j        j        t          d¦  «        k    }| j        j        }|r|st          |t          ¦  «        rdS | j        j        r| j        j	        j
        rdS | j        j        }|�&t          j        ¦   «         }|| j        j        z
  |k    S |}t          | j        j        t           ¦  «        r|r| j        dz   | j        j        z  dk    }n[| j        j        t          d¦  «        k    r>| j        j        €J ‚| j        j        €| j        n| j        }|dz   | j        j        z  dk    }|S )z#Decide if we should run validation.Fr|   TNrI   r   )r  r.   Úval_check_batchr‰   r9   r¸   r~   r   rb   rc   re   Ú_val_check_time_intervalÚtimeÚ	monotonicÚ_last_val_timeÚlimit_train_batchesÚintrP   r  rM   )rF   ri   Úis_infinite_datasetr¸   rþ   ÚnowÚis_val_check_batchÚcurrent_iterations           r*   rÀ   z'_TrainingEpochLoop._should_check_val_fx  sg  € à×+Ò+Ñ-Ô-ð 	Ø�5ð #œlÔ:½eÀE¹l¼lÒJÐØÔ+Ô9ˆØð 	Ð1ð 	µZÀÕNhÑ5iÔ5ið 	Ø�4àŒ<Ô#ð 	¨¬Ô(=Ô(Mð 	ð �4à”<Ô8ˆØÐÝ”.Ñ"Ô"ˆCà˜œÔ4Ñ4¸Ò@Ð@à*ÐÝ�d”lÔ6½Ñ<Ô<ð 	]ÐATð 	]Ø"&¤.°1Ñ"4¸¼Ô8XÑ!XÐ\]Ò!]ÐÐØŒ\Ô)­U°5©\¬\Ò9Ð9à”<Ô/Ð;Ð;Ð;ð 9=¼Ô8\Ð8d Ô 4Ð 4ÐjnÔjxÐØ"3°aÑ"7¸4¼<Ô;WÑ!WÐ[\Ò!\Ðà!Ð!r)   c                 óh   — | j         j        r#| j         j        D ]}|                     ¦   «          ŒdS dS )zFlushes loggers to disk.N)r.   rb   ÚloggersÚsave)rF   Úloggers     r*   ré   z3_TrainingEpochLoop._save_loggers_on_train_batch_end4  sF   € àŒ<Ô#ð 	Øœ,Ô.ð ð �Ø—’‘”��ð	ð 	ðð r)   rä   rÞ   rP   c                 ór   — ||d<   t          | j        j        d¦  «        }t          |dd¬¦  «        r||d<   |S )aD  Helper method to build the arguments for the current step.

        Args:
            kwargs: The kwargs passed down to the hooks.
            batch: The current batch to run through the step.
            batch_idx: the index of the current batch.

        Returns:
            The kwargs passed down to the hooks.

        rÞ   Útraining_steprP   é   )Úmin_args)r¨   r.   rS   r   )rF   rä   rÞ   rP   Útraining_step_fxs        r*   rÕ   z _TrainingEpochLoop._build_kwargs:  sK   € ð  ˆˆw‰Ý" 4¤<Ô#@À/ÑRÔRÐõ &Ð&6¸ÈaÐPÑPÔPð 	,Ø"+ˆF�;ÑØˆr)   )Nr-   )r1   N)/r"   r#   r$   Ú__doc__r   r  r8   ÚpropertyrM   rP   rX   Úboolr\   r`   rh   r   rr   r    r!   rx   rz   rk   rl   r—   r™   r³   rm   rn   rî   r   Údictrð   rô   rö   rZ   r„   ÚstrrÙ   r   r   r  r  rÀ   ré   r   rÕ   Ú__classcell__)rG   s   @r*   r,   r,   3   s©  ø€ € € € € ðð ð*$ð $ ð $¸À#¼ð $ÐZ]ð $Ðgkð $ð $ð $ð $ð $ð $ð2 ð3 ð 3ð 3ð 3ñ „Xð3ð ð5˜3ð 5ð 5ð 5ñ „Xð5ð ðL˜Sð Lð Lð Lñ „XðLð ðF 4ð Fð Fð Fñ „XðFð ð= Tð =ð =ð =ñ „Xð=ð ð�dð ð ð ñ „Xðð$
) ð 
)°ð 
)ð 
)ð 
)ð 
)ð ðP¨dð Pð Pð Pñ „XðPð ðE 4ð Eð Eð Eñ „XðEð-ð -ð -ð -ð(0ð 0ð 0ð 0ð.7ð .7ð .7ð .7ð`;¨ð ;¸$ð ;ð ;ð ;ð ;ðZð Zð Zð ZðYð Yð Yð Yð%ð %ð %ð %ðn> Lð n>°Tð n>ð n>ð n>ð n>ð`$#¨<ð $#¸Dð $#ð $#ð $#ð $#ðL!ð !ð !ð !ð ð Dð ð ð ð ð ñ „Xðð
 ðP¨Tð P°dð Pð Pð Pñ „XðPð]¨dð ]ð ]ð ]ð ]ðL¨Dð Lð Lð Lð Lð
M Dð Mð Mð Mð Mðl¨Sð lÈTð lÐVZð lð lð lð lð7>¨sð 7>Ètð 7>ÐX\ð 7>ð 7>ð 7>ð 7>ðr6 cð 6¨h°s¬mð 6ð 6ð 6ð 6ð
¨ð 
ð 
ð 
ð 
ð!"°ð !"À$ð !"ð !"ð !"ð !"ðFð ð ð ð Kð ¸ð Èð ÐP[ð ð ð ð ð ð ð ð r)   r,   )8r¯   rŠ   r  Úcollectionsr   Údataclassesr   Útypingr   r   r   r¦   Útyping_extensionsr   Úpytorch_lightningÚplÚ lightning_fabric.utilities.typesr	   Ú#lightning_fabric.utilities.warningsr
   r   Ú pytorch_lightning.loops.fetchersr   r   Ú$pytorch_lightning.loops.optimizationr   r   Ú.pytorch_lightning.loops.optimization.automaticr   Ú_OPTIMIZER_LOOP_OUTPUTS_TYPEÚ+pytorch_lightning.loops.optimization.manualÚ_MANUAL_LOOP_OUTPUTS_TYPEÚ pytorch_lightning.loops.progressr   r   Ú!pytorch_lightning.loops.utilitiesr   Úpytorch_lightning.trainerr   Ú<pytorch_lightning.trainer.connectors.logger_connector.resultr   Ú pytorch_lightning.trainer.statesr   r   Ú&pytorch_lightning.utilities.exceptionsr   r   Ú%pytorch_lightning.utilities.rank_zeror   r   Ú+pytorch_lightning.utilities.signature_utilsr   Ú_BATCH_OUTPUTS_TYPEr   Ú_Loopr,   r(   r)   r*   ú<module>rK     sK  ðð Ð Ð Ð Ø €€€Ø €€€Ø #Ð #Ð #Ð #Ð #Ð #Ø !Ð !Ð !Ð !Ð !Ð !Ø 'Ð 'Ð 'Ð 'Ð 'Ð 'Ð 'Ð 'Ð 'Ð 'à €€€Ø &Ð &Ð &Ð &Ð &Ð &à Ð Ð Ð Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø CÐ CÐ CÐ CÐ CÐ CØ #Ð #Ð #Ð #Ð #Ð #Ø UÐ UÐ UÐ UÐ UÐ UÐ UÐ UØ \Ð \Ð \Ð \Ð \Ð \Ð \Ð \Ø hÐ hÐ hÐ hÐ hÐ hØ bÐ bÐ bÐ bÐ bÐ bØ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OØ CÐ CÐ CÐ CÐ CÐ CØ *Ð *Ð *Ð *Ð *Ð *Ø ZÐ ZÐ ZÐ ZÐ ZÐ ZØ DÐ DÐ DÐ DÐ DÐ DÐ DÐ DØ ^Ð ^Ð ^Ð ^Ð ^Ð ^Ð ^Ð ^Ø NÐ NÐ NÐ NÐ NÐ NÐ NÐ NØ RÐ RÐ RÐ RÐ RÐ Rà˜uÐ%AÐC\Ð%\Ô]Ô^Ð ð ð,ð ,ð ,ð ,ð ,ñ ,ô ,ñ „ð,ðYð Yð Yð Yð Y˜œñ Yô Yð Yð Yð Yr)   