§
    “©ºj€É  ã                   óZ  — d Z ddlZddlZddlZddlZddlZddlZddlmZ ddl	m
Z
 ddlmZ ddlmZmZmZmZ ddlmZ ddlZddlZddlmZ dd	lmZ ddlZdd
lmZmZmZ ddlm Z  ddl!m"Z" ddl#m$Z$ ddl%m&Z&m'Z'm(Z( ddl)m*Z*  ej+        e,¦  «        Z- e&¦   «         Z. G d„ de"¦  «        Z/dS )z`
Model Checkpointing
===================

Automatically save model checkpoints during training.
é    N)Údeepcopy)Ú	timedelta)ÚPath)ÚAnyÚLiteralÚOptionalÚUnion)Úproxy)ÚTensor)Úoverride)Ú_is_dirÚ_is_local_file_protocolÚget_filesystem)Ú_PATH)Ú
Checkpoint)ÚMisconfigurationException)ÚWarningCacheÚrank_zero_infoÚrank_zero_warn)ÚSTEP_OUTPUTc                   óê  ‡ — e Zd ZdZdZdZdZdZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dUde	e
         de	e         de	e         dede	eeed         f                  dededededede	e         de	e         de	e         de	e         defˆ fd„Zeedefd„¦   «         ¦   «         Zeddd d!d"eddfd#„¦   «         ZedVd$„¦   «         Zeddd d!d%ed&ed'eddfd(„¦   «         ZedVd)„¦   «         ZedVd*„¦   «         Zeddd d!d+eddfd,„¦   «         ZdVd-„Zedeeef         fd.„¦   «         Z ed/eeef         ddfd0„¦   «         Z!ddd1eee"f         ddfd2„Z#ddd3eddfd4„Z$e%ddd3ed5eddfd6„¦   «         Z&dddefd7„Z'dddefd8„Z(dddefd9„Z)dWd:„Z*de	e
         de	e         ddfd;„Z+deddfd<„Z,de	e         de	e         de	e         ddfd=„Z-ede	e         fd>„¦   «         Z.dXddd?e	e"         defd@„Z/	 	 dYde	e         dAeee"f         dBe	e         dedef
dC„Z0	 	 	 dZdAeee"f         de	e         dDe	e         dBe	e         def
dE„Z1ddde
fdF„Z2ddde3e         fdG„Z4de
ddfdH„Z5	 dXd1eee"f         dddIe	e         defdJ„Z6dddeee"f         fdK„Z7ddd1eee"f         ddfdL„Z8ddd1eee"f         ddfdM„Z9ddd1eee"f         ddfdN„Z:d?e"ddd1eee"f         ddfdO„Z;dXd3e	e
         ddfdP„Z<d3e
dddefdQ„Z=dddRed?edefdS„Z>ddd3eddfdT„Z?ˆ xZ@S )[ÚModelCheckpointa0  Save the model after every epoch by monitoring a quantity. Every logged metrics are passed to the
    :class:`~pytorch_lightning.loggers.logger.Logger` for the version it gets saved in the same directory as the
    checkpoint.

    After training finishes, use :attr:`best_model_path` to retrieve the path to the
    best checkpoint file and :attr:`best_model_score` to get its score.

    .. note::
        When using manual optimization with ``every_n_train_steps``, you should save the model state
        in your ``training_step`` before the optimizer step if you want the checkpoint to reflect
        the pre-optimization state. Example:

        .. code-block:: python

            def training_step(self, batch, batch_idx):
                # ... forward pass, loss calculation, backward pass ...

                # Save model state before optimization
                if not hasattr(self, 'saved_models'):
                    self.saved_models = {}
                self.saved_models[batch_idx] = {
                    k: v.detach().clone()
                    for k, v in self.layer.state_dict().items()
                }

                # Then perform optimization
                optimizer.zero_grad()
                self.manual_backward(loss)
                optimizer.step()

                # Optional: Clean up old states to save memory
                if batch_idx > 10:  # Keep last 10 states
                    del self.saved_models[batch_idx - 10]

    Args:
        dirpath: Directory to save the model file.
            Example: ``dirpath='my/path/'``.

            .. warning::
                In a distributed environment like DDP, it's recommended to provide a `dirpath` to avoid race conditions.
                When using manual optimization with ``every_n_train_steps``, make sure to save the model state
                in your training loop as shown in the example above.

            Can be remote file paths such as `s3://mybucket/path/` or 'hdfs://path/'
            (default: ``None``). If dirpath is ``None``, we only keep the ``k`` best checkpoints
            in memory, and do not save anything to disk.

        filename: Checkpoint filename. Can contain named formatting options to be auto-filled.
            If no name is provided, it will be ``None`` and the checkpoint will be saved to
            ``{epoch}``.and if the Trainer uses a logger, the path will also contain logger name and version.

        filename: checkpoint filename. Can contain named formatting options to be auto-filled.

            Example::

                # save any arbitrary metrics like `val_loss`, etc. in name
                # saves a file like: my/path/epoch=2-val_loss=0.02-other_metric=0.03.ckpt
                >>> checkpoint_callback = ModelCheckpoint(
                ...     dirpath='my/path',
                ...     filename='{epoch}-{val_loss:.2f}-{other_metric:.2f}'
                ... )

            By default, filename is ``None`` and will be set to ``'{epoch}-{step}'``, where "epoch" and "step" match
            the number of finished epoch and optimizer steps respectively.
        monitor: quantity to monitor. By default it is ``None`` which saves a checkpoint only for the last epoch.
        verbose: verbosity mode. Default: ``False``.
        save_last: When ``True``, saves a `last.ckpt` copy whenever a checkpoint file gets saved. Can be set to
            ``'link'`` on a local filesystem to create a symbolic link. This allows accessing the latest checkpoint
            in a deterministic manner. Default: ``None``.
        save_top_k: if ``save_top_k == k``,
            the best k models according to the quantity monitored will be saved.
            If ``save_top_k == 0``, no models are saved.
            If ``save_top_k == -1``, all models are saved.
            Please note that the monitors are checked every ``every_n_epochs`` epochs.
            If ``save_top_k >= 2`` and the callback is called multiple times inside an epoch, and the filename remains
            unchanged, the name of the saved file will be appended with a version count starting with ``v1`` to avoid
            collisions unless ``enable_version_counter`` is set to False. The version counter is unrelated to the top-k
            ranking of the checkpoint, and we recommend formatting the filename to include the monitored metric to avoid
            collisions.
        save_on_exception: Whether to save a checkpoint when an exception is raised. Default: ``False``.
        mode: one of {min, max}.
            If ``save_top_k != 0``, the decision to overwrite the current save file is made
            based on either the maximization or the minimization of the monitored quantity.
            For ``'val_acc'``, this should be ``'max'``, for ``'val_loss'`` this should be ``'min'``, etc.
        auto_insert_metric_name: When ``True``, the checkpoints filenames will contain the metric name.
            For example, ``filename='checkpoint_{epoch:02d}-{acc:02.0f}`` with epoch ``1`` and acc ``1.12`` will resolve
            to ``checkpoint_epoch=01-acc=01.ckpt``. Is useful to set it to ``False`` when metric names contain ``/``
            as this will result in extra folders.
            For example, ``filename='epoch={epoch}-step={step}-val_acc={val/acc:.2f}', auto_insert_metric_name=False``
        save_weights_only: if ``True``, then only the model's weights will be
            saved. Otherwise, the optimizer states, lr-scheduler states, etc are added in the checkpoint too.
        every_n_train_steps: How many training steps to wait before saving a checkpoint. This does not take into account
            the steps of the current epoch. If ``every_n_train_steps == None or every_n_train_steps == 0``,
            no checkpoints
            will be saved during training. Mutually exclusive with ``train_time_interval`` and ``every_n_epochs``.

            .. note::
                When using with manual optimization, the checkpoint will be saved after the optimizer step by default.
                To save the model state before the optimizer step, you need to save the model state in your
                ``training_step`` before calling ``optimizer.step()``. See the class docstring for an example.
        train_time_interval: Checkpoints are monitored at the specified time interval.
            For all practical purposes, this cannot be smaller than the amount
            of time it takes to process a single training batch. This is not
            guaranteed to execute at the exact time specified, but should be close.
            This must be mutually exclusive with ``every_n_train_steps`` and ``every_n_epochs``.
        every_n_epochs: Number of epochs between checkpoints.
            This value must be ``None`` or non-negative.
            To disable saving top-k checkpoints, set ``every_n_epochs = 0``.
            This argument does not impact the saving of ``save_last=True`` checkpoints.
            If all of ``every_n_epochs``, ``every_n_train_steps`` and
            ``train_time_interval`` are ``None``, we save a checkpoint at the end of every epoch
            (equivalent to ``every_n_epochs = 1``).
            If ``every_n_epochs == None`` and either ``every_n_train_steps != None`` or ``train_time_interval != None``,
            saving at the end of each epoch is disabled
            (equivalent to ``every_n_epochs = 0``).
            This must be mutually exclusive with ``every_n_train_steps`` and ``train_time_interval``.
            Setting both ``ModelCheckpoint(..., every_n_epochs=V, save_on_train_epoch_end=False)`` and
            ``Trainer(max_epochs=N, check_val_every_n_epoch=M)``
            will only save checkpoints at epochs 0 < E <= N
            where both values for ``every_n_epochs`` and ``check_val_every_n_epoch`` evenly divide E.
        save_on_train_epoch_end: Whether to run checkpointing at the end of the training epoch.
            If ``True``, checkpoints are saved at the end of every training epoch.
            If ``False``, checkpoints are saved at the end of validation.
            If ``None`` (default), checkpointing behavior is determined based on training configuration.
            If ``val_check_interval`` is a str, dict, or `timedelta` (time-based), checkpointing is performed after
            validation.
            If ``check_val_every_n_epoch != 1``, checkpointing will not be performed at the end of
            every training epoch. If there are no validation batches of data, checkpointing will occur at the
            end of the training epoch. If there is a non-default number of validation runs per training epoch
            (``val_check_interval != 1``), checkpointing is performed after validation.
        enable_version_counter: Whether to append a version to the existing file name.
            If ``False``, then the checkpoint files will be overwritten.

    Note:
        For extra customization, ModelCheckpoint includes the following attributes:

        - ``CHECKPOINT_JOIN_CHAR = "-"``
        - ``CHECKPOINT_EQUALS_CHAR = "="``
        - ``CHECKPOINT_NAME_LAST = "last"``
        - ``FILE_EXTENSION = ".ckpt"``
        - ``STARTING_VERSION = 1``

        For example, you can change the default last checkpoint name by doing
        ``checkpoint_callback.CHECKPOINT_NAME_LAST = "{epoch}-last"``

        If you want to checkpoint every N hours, every M train batches, and/or every K val epochs,
        then you should create multiple ``ModelCheckpoint`` callbacks.

        If the checkpoint's ``dirpath`` changed from what it was before while resuming the training,
        only ``best_model_path`` will be reloaded and a warning will be issued.

        If you provide a ``filename`` on a mounted device where changing permissions is not allowed (causing ``chmod``
        to raise a ``PermissionError``), install `fsspec>=2025.5.0`. Then the error is caught, the file's permissions
        remain unchanged, and the checkpoint is still saved. Otherwise, no checkpoint will be saved and training stops.

    Raises:
        MisconfigurationException:
            If ``save_top_k`` is smaller than ``-1``,
            if ``monitor`` is ``None`` and ``save_top_k`` is none of ``None``, ``-1``, and ``0``, or
            if ``mode`` is none of ``"min"`` or ``"max"``.
        ValueError:
            If ``trainer.save_checkpoint`` is ``None``.

    Example::

        >>> from pytorch_lightning import Trainer
        >>> from pytorch_lightning.callbacks import ModelCheckpoint

        # saves checkpoints to 'my/path/' at every epoch
        >>> checkpoint_callback = ModelCheckpoint(dirpath='my/path/')
        >>> trainer = Trainer(callbacks=[checkpoint_callback])

        # save epoch and val_loss in name
        # saves a file like: my/path/sample-mnist-epoch=02-val_loss=0.32.ckpt
        >>> checkpoint_callback = ModelCheckpoint(
        ...     monitor='val_loss',
        ...     dirpath='my/path/',
        ...     filename='sample-mnist-{epoch:02d}-{val_loss:.2f}'
        ... )

        # save epoch and val_loss in name, but specify the formatting yourself (e.g. to avoid problems with Tensorboard
        # or other loggers, due to the presence of characters like '=' or '/')
        # saves a file like: my/path/sample-mnist-epoch02-val_loss0.32.ckpt
        >>> checkpoint_callback = ModelCheckpoint(
        ...     monitor='val/loss',
        ...     dirpath='my/path/',
        ...     filename='sample-mnist-epoch{epoch:02d}-val_loss{val/loss:.2f}',
        ...     auto_insert_metric_name=False
        ... )

        # retrieve the best checkpoint after training
        >>> checkpoint_callback = ModelCheckpoint(dirpath='my/path/')
        >>> trainer = Trainer(callbacks=[checkpoint_callback])
        >>> model = ...  # doctest: +SKIP
        >>> trainer.fit(model)  # doctest: +SKIP
        >>> print(checkpoint_callback.best_model_path)  # doctest: +SKIP

    .. tip:: Saving and restoring multiple checkpoint callbacks at the same time is supported under variation in the
        following arguments:

        *monitor, mode, every_n_train_steps, every_n_epochs, train_time_interval*

        Read more: :ref:`Persisting Callback State <extensions/callbacks_state:save callback state>`

    ú-ú=Úlastz.ckpté   NFÚminTÚdirpathÚfilenameÚmonitorÚverboseÚ	save_lastÚlinkÚ
save_top_kÚsave_on_exceptionÚsave_weights_onlyÚmodeÚauto_insert_metric_nameÚevery_n_train_stepsÚtrain_time_intervalÚevery_n_epochsÚsave_on_train_epoch_endÚenable_version_counterc                 ó  •— t          ¦   «                              ¦   «          || _        || _        || _        || _        || _        || _        |
| _        || _	        || _
        d| _        d | _        d | _        i | _        d| _        d | _        d| _        d| _        d| _        d| _        |  |  |                      |	¦  «         |                      ||¦  «         |                      |||¦  «         |                      ¦   «          d S )Nr   Ú F)ÚsuperÚ__init__r    r!   r"   r$   r%   r&   r(   Ú_save_on_train_epoch_endÚ_enable_version_counterÚ_last_global_step_savedÚ_last_time_checkedÚcurrent_scoreÚbest_k_modelsÚkth_best_model_pathÚbest_model_scoreÚbest_model_pathÚlast_model_pathÚ_last_checkpoint_savedÚ_defer_save_until_validationÚ#_ModelCheckpoint__init_monitor_modeÚ_ModelCheckpoint__init_ckpt_dirÚ_ModelCheckpoint__init_triggersÚ-_ModelCheckpoint__validate_init_configuration)Úselfr   r   r    r!   r"   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   Ú	__class__s                   €ú|/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/pytorch_lightning/callbacks/model_checkpoint.pyr1   zModelCheckpoint.__init__  s  ø€ õ$ 	‰Œ×ÒÑÔÐØˆŒØˆŒØ"ˆŒØ$ˆŒØ!2ˆÔØ!2ˆÔØ'>ˆÔ$Ø(?ˆÔ%Ø'=ˆÔ$Ø'(ˆÔ$Ø37ˆÔØ/3ˆÔØ02ˆÔØ#%ˆÔ Ø26ˆÔØ!ˆÔØ!ˆÔØ&(ˆÔ#ð 38ˆÔ)àÐØÐ%Ø× Ò  Ñ&Ô&Ð&Ø×Ò˜W hÑ/Ô/Ð/Ø×ÒÐ0°.ÐBUÑVÔVÐVØ×*Ò*Ñ,Ô,Ð,Ð,Ð,ó    Úreturnc                 óh   — |                       | j        | j        | j        | j        | j        ¬¦  «        S )N)r    r'   r)   r+   r*   )Ú_generate_state_keyr    r'   Ú_every_n_train_stepsÚ_every_n_epochsÚ_train_time_interval©rB   s    rD   Ú	state_keyzModelCheckpoint.state_key7  s=   € ð ×'Ò'Ø”LØ”Ø $Ô 9ØÔ/Ø $Ô 9ð (ñ 
ô 
ð 	
rE   Útrainerú
pl.TrainerÚ	pl_moduleúpl.LightningModuleÚstagec                 ó^  — |                       |¦  «        }|j                             |¦  «        }|| _        t	          | j        pd¦  «        | _        |j        r |dk    r|                      | j        ¦  «         | j        dk    r't          | j        ¦  «        st          d|› d�¦  «        ‚d S d S )Nr/   Úfitr#   zY`ModelCheckpoint(save_last='link')` is only supported for local file paths, got `dirpath=z`.)Ú"_ModelCheckpoint__resolve_ckpt_dirÚstrategyÚ	broadcastr   r   Ú_fsÚis_global_zeroÚ'_ModelCheckpoint__warn_if_dir_not_emptyr"   r   Ú
ValueError)rB   rN   rP   rR   r   s        rD   ÚsetupzModelCheckpoint.setupB  s½   € à×)Ò)¨'Ñ2Ô2ˆØÔ"×,Ò,¨WÑ5Ô5ˆØˆŒÝ! $¤,Ð"4°"Ñ5Ô5ˆŒØÔ!ð 	7 e¨u¢n nØ×(Ò(¨¬Ñ6Ô6Ð6ØŒ>˜VÒ#Ð#Õ,CÀDÄLÑ,QÔ,QÐ#ÝØwÐlsÐwÐwÐwñô ð ð $Ð#Ð#Ð#rE   c                 ó6   — t          j        ¦   «         | _        d S ©N)ÚtimeÚ	monotonicr5   )rB   rN   rP   s      rD   Úon_train_startzModelCheckpoint.on_train_startO  s   € å"&¤.Ñ"2Ô"2ˆÔÐÐrE   ÚoutputsÚbatchÚ	batch_idxc                 ó�  — |j         �sŒ| j        dk     s|j        | j        z  dk    rdS |                      |¦  «        rdS |                      |¦  «        }| j        �| j        |vr	d| _        dS t          |d¦  «        �r¬t          |j	        t          ¦  «        �r‘|j	        �r‰t          |d¦  «        �rxt          |j        t          j        j        ¦  «        �rS|j	        }|sdS t          |                     ¦   «         ¦  «        }t          j        ¦   «         5  d„ |j                             ¦   «                              ¦   «         D ¦   «         }		 ||         }
t          |
t          ¦  «        st)          d¦  «        ‚|j                             |
¦  «         |                      ||¦  «         |                      ||¦  «         t1          j        ¦   «         | _        |j                             |	¦  «         n# |j                             |	¦  «         w xY w	 ddd¦  «         n# 1 swxY w Y   nh|j         s|j        rt9          d	t:          ¬
¦  «         |                      ||¦  «         |                      ||¦  «         t1          j        ¦   «         | _        dS |                      |¦  «        }| j        dk     p|j        | j        z  dk    }| j        }d}t1          j        ¦   «         }|�@| j        }|du p||z
  |                     ¦   «         k     }|j          !                    |¦  «        }|r|rdS |s|| _        |                      |¦  «        }| j        �| j        |vr	d| _        dS | j        �¬|  "                    |¦  «        s—tG          |j$        j%        j&        dd¦  «        rwd}tG          |dd¦  «        rYt          |j'        tP          ¦  «        rtS          |j'        ¦  «        n|j'        }|r"|dk    r|j*        }|�|j+        dz   |z  dk    rd}|r	d| _        dS |rdS |                      ||¦  «         |                      ||¦  «         dS )zTSave checkpoint on train batch end if we meet the criteria for `every_n_train_steps`r   r   NTÚsaved_modelsÚlayerc                 ób   — i | ],\  }}||                      ¦   «                              ¦   «         “Œ-S © )ÚdetachÚclone©Ú.0ÚkÚvs      rD   ú
<dictcomp>z6ModelCheckpoint.on_train_batch_end.<locals>.<dictcomp>  s2   € Ð%mÐ%mÐ%mÁÀÀ1 a¨¯ª©¬×)9Ò)9Ñ);Ô);Ð%mÐ%mÐ%mrE   z&Saved model state must be a dictionaryaÚ  Using ModelCheckpoint with manual optimization and every_n_train_steps, but no pre-optimization state was saved. The checkpoint will contain the model state AFTER optimization. To save the pre-optimization state, save the model state in training_step before optimizer.step(). Example:
def training_step(self, batch, batch_idx):
    # ... forward pass, loss calculation, backward pass ...
    # Save model state before optimization
    if not hasattr(self, 'saved_models'):
        self.saved_models = {}
    self.saved_models[batch_idx] = {
        k: v.detach().clone() for k, v in self.layer.state_dict().items()
    }
    # Then perform optimization
    optimizer.zero_grad()
    self.manual_backward(loss)
    optimizer.step())ÚcategoryÚis_last_batchFÚenable_validation),Úautomatic_optimizationrI   Úglobal_stepÚ_should_skip_saving_checkpointÚ_monitor_candidatesr    r=   ÚhasattrÚ
isinstancerf   Údictrg   ÚtorchÚnnÚModuleÚmaxÚkeysÚno_gradÚ
state_dictÚitemsÚ	TypeErrorÚload_state_dictÚ_save_topk_checkpointÚ_save_last_checkpointr_   r`   r5   rY   r   ÚUserWarningrK   Útotal_secondsrV   rW   Ú_should_save_on_train_epoch_endÚgetattrÚfit_loopÚ
epoch_loopÚbatch_progressÚnum_val_batchesÚlistÚsumÚcheck_val_every_n_epochÚcurrent_epoch)rB   rN   rP   rb   rc   rd   Úmonitor_candidatesrf   Úlatest_stepÚoriginal_stateÚsaved_stateÚskip_due_to_stateÚ
skip_batchr*   Ú	skip_timeÚnowÚprev_time_checkÚwill_run_valrŽ   Úcves                       rD   Úon_train_batch_endz"ModelCheckpoint.on_train_batch_endS  s  € ð Ô/ñ K	àÔ(¨1Ò,Ð,°Ô1DÀtÔG`Ñ1`ÐdeÒ1eÐ1eØ�ð ×2Ò2°7Ñ;Ô;ð Ø�ð "&×!9Ò!9¸'Ñ!BÔ!BÐØŒ|Ð'¨D¬LÐ@RÐ,RÐ,RØ48�Ô1Ø�õ
 ˜	 >Ñ2Ô2ñ9;å˜yÔ5µtÑ<Ô<ñ9;ð Ô*ñ9;õ ˜I wÑ/Ô/ñ	9;õ
 ˜yœµ´´Ñ@Ô@ñ9;ð  )Ô5�Ø#ð Ø�Få! ,×"3Ò"3Ñ"5Ô"5Ñ6Ô6�å”]‘_”_ð Hð Hà%mÐ%mÈ	Ì×HbÒHbÑHdÔHd×HjÒHjÑHlÔHlÐ%mÑ%mÔ%m�NðHà&2°;Ô&?˜Ý)¨+µtÑ<Ô<ð VÝ"+Ð,TÑ"UÔ"UÐUà!œ×7Ò7¸ÑDÔDÐDà×2Ò2°7Ð<NÑOÔOÐOØ×2Ò2°7Ð<NÑOÔOÐOÝ26´.Ñ2BÔ2B˜Ô/ð "œ×7Ò7¸ÑGÔGÐGÐGø˜	œ×7Ò7¸ÑGÔGÐGÐGøøøÐGð!Hð Hð Hñ Hô Hð Hð Hð Hð Hð Hð Høøøð Hð Hð Hð Høð& !Ô7ð ¸GÔ<Rð Ý"ð/õ$ "-ð'ñ ô ð ð* ×*Ò*¨7Ð4FÑGÔGÐGØ×*Ò*¨7Ð4FÑGÔGÐGÝ*.¬.Ñ*:Ô*:�Ô'ØˆFð !×?Ò?ÀÑHÔHÐØÔ.°Ò2Ðl°wÔ7JÈTÔMfÑ7fÐjkÒ7kˆ
à"Ô7ÐØˆ	ÝŒnÑÔˆàÐ*Ø"Ô5ˆOØ'¨4Ð/Ðp°C¸/Ñ4IÐM`×MnÒMnÑMpÔMpÒ3pˆIð  Ô(×2Ò2°9Ñ=Ô=ˆIàð 	˜)ð 	ØˆFØð 	*Ø&)ˆDÔ#à!×5Ò5°gÑ>Ô>Ðð Œ<Ð#¨¬Ð<NÐ(NÐ(Nà04ˆDÔ-ØˆFð ŒLÐ$Ø×8Ò8¸ÑAÔAð %å˜Ô(Ô3ÔBÀOÐUZÑ[Ô[ð %ð
 !ˆLÝ�wÐ 3°UÑ;Ô;ð 	,õ " 'Ô"9½4Ñ@Ô@ð1•C˜Ô/Ñ0Ô0Ð0à Ô0ð  ð
 #ð , ¸Ò':Ð':Ø!Ô9�CØ�{¨Ô(=ÀÑ(AÀSÑ'HÈAÒ'MÐ'MØ'+˜àð Ø48�Ô1Ø�ð ð 	ØˆFà×"Ò" 7Ð,>Ñ?Ô?Ð?Ø×"Ò" 7Ð,>Ñ?Ô?Ð?Ð?Ð?s+   Ä6HÅB
G3ÇHÇ3HÈHÈH#È&H#c                 ór  — |                       |¦  «        s›|                      |¦  «        rˆ|                      |¦  «        }| j        dk    r,|j        dz   | j        z  dk    r|                      ||¦  «         | j        |j        k    s| j        dk    r#| j	        r|  
                    ||¦  «         dS dS dS dS dS )z3Save a checkpoint at the end of the training epoch.r   r   r#   N)rv   r‰   rw   rJ   r’   r…   r4   ru   r"   r<   r†   ©rB   rN   rP   r“   s       rD   Úon_train_epoch_endz"ModelCheckpoint.on_train_epoch_endç  sï   € ð ×2Ò2°7Ñ;Ô;ð 	HÀ×@dÒ@dÐelÑ@mÔ@mð 	HØ!%×!9Ò!9¸'Ñ!BÔ!BÐØÔ# qÒ(Ð(¨gÔ.CÀaÑ.GÈ4ÔK_Ñ-_ÐcdÒ-dÐ-dØ×*Ò*¨7Ð4FÑGÔGÐGàÔ+¨wÔ/BÒBÐBØ” &Ò(Ð(¨TÔ-HÐ(à×*Ò*¨7Ð4FÑGÔGÐGÐGÐGð	Hð 	Hð 	Hð 	Hð )Ð(Ð(Ð(rE   c                 óê  — |                       |¦  «        s×|                      |¦  «        sÄ|                      |¦  «        }| j        r5|                      ||¦  «         |                      ||¦  «         d| _        dS | j        dk    r,|j        dz   | j        z  dk    r|                      ||¦  «         | j        |j	        k    s| j
        dk    r#| j        r|                      ||¦  «         dS dS dS dS dS )z5Save a checkpoint at the end of the validation stage.FNr   r   r#   )rv   r‰   rw   r=   r…   r†   rJ   r’   r4   ru   r"   r<   r    s       rD   Úon_validation_endz!ModelCheckpoint.on_validation_endô  s7  € ð ×2Ò2°7Ñ;Ô;ð 	HÀD×DhÒDhÐipÑDqÔDqð 	HØ!%×!9Ò!9¸'Ñ!BÔ!BÐð Ô0ð Ø×*Ò*¨7Ð4FÑGÔGÐGØ×*Ò*¨7Ð4FÑGÔGÐGØ49�Ô1Ø�àÔ# qÒ(Ð(¨gÔ.CÀaÑ.GÈ4ÔK_Ñ-_ÐcdÒ-dÐ-dØ×*Ò*¨7Ð4FÑGÔGÐGàÔ+¨wÔ/BÒBÐBØ” &Ò(Ð(¨TÔ-HÐ(à×*Ò*¨7Ð4FÑGÔGÐGÐGÐGð!	Hð 	Hð 	Hð 	Hð )Ð(Ð(Ð(rE   Ú	exceptionc                 óP  — |                       |¦  «        sdS |                      |¦  «        }|                      |¬¦  «        }|                      ||¦  «         |                      ||¦  «         t          dt          |¦  «        j        › dt          |¦  «        › d|› �¦  «         dS )z.Save a checkpoint when an exception is raised.N)ÚmetricszAn z& was raised with message:             z, saved checkpoint to )	Ú_should_save_on_exceptionrw   Úformat_checkpoint_nameÚ_save_checkpointr†   r   ÚtypeÚ__name__Ústr)rB   rN   rP   r¤   r“   Úfilepaths         rD   Úon_exceptionzModelCheckpoint.on_exception	  sÍ   € ð ×-Ò-¨gÑ6Ô6ð 	ØˆFØ!×5Ò5°gÑ>Ô>ÐØ×.Ò.Ð7IÐ.ÑJÔJˆØ×Ò˜g xÑ0Ô0Ð0Ø×"Ò" 7Ð,>Ñ?Ô?Ð?Ýð>•$�y‘/”/Ô*ð >ð >Ý�‰^Œ^ð>ð >Ø3;ð>ð >ñ	
ô 	
ð 	
ð 	
ð 	
rE   c                 ó€   — | j         r4| j        s/|                      |¦  «        }|                      ||¦  «         dS dS dS )z4Ensure save_last=True is applied when training ends.N)r"   r<   rw   r†   r    s       rD   Úon_train_endzModelCheckpoint.on_train_end  sa   € àŒ>ð 	D $Ô"=ð 	DØ!%×!9Ò!9¸'Ñ!BÔ!BÐØ×&Ò& wÐ0BÑCÔCÐCÐCÐCð	Dð 	Dð 	Dð 	DrE   c           
      ót   — | j         | j        | j        | j        | j        | j        | j        | j        | j        dœ	S )N©	r    r9   r:   r6   r   r7   r8   Ú	kth_valuer;   r²   rL   s    rD   r�   zModelCheckpoint.state_dict  sF   € ð ”|Ø $Ô 5Ø#Ô3Ø!Ô/Ø”|Ø!Ô/Ø#'Ô#;ØœØ#Ô3ð

ð 

ð 
	
rE   r�   c                 óÈ  — |                      d| j        ¦  «        }| j        |k    rŽ|d         | _        |                      d| j        ¦  «        | _        |                      d| j        ¦  «        | _        |                      d| j        ¦  «        | _        |                      d| j        ¦  «        | _        n t          j        d|›d| j        ›d	�¦  «         |d
         | _	        d S )Nr   r9   r8   r³   r7   r;   zThe dirpath has changed from z to z¥, therefore `best_model_score`, `kth_best_model_path`, `kth_value`, `last_model_path` and `best_k_models` won't be reloaded. Only `best_model_path` will be reloaded.r:   )
Úgetr   r9   r8   r³   r7   r;   ÚwarningsÚwarnr:   )rB   r�   Údirpath_from_ckpts      rD   r„   zModelCheckpoint.load_state_dict+  sð   € à&ŸNšN¨9°d´lÑCÔCÐàŒ<Ð,Ò,Ð,Ø$.Ð/AÔ$BˆDÔ!Ø'1§~¢~Ð6KÈTÔMeÑ'fÔ'fˆDÔ$Ø'Ÿ^š^¨K¸¼ÑHÔHˆDŒNØ!+§¢°ÀÔASÑ!TÔ!TˆDÔØ#-§>¢>Ð2CÀTÔEYÑ#ZÔ#ZˆDÔ Ð åŒMð_Ð0Að _ð _ÈÌð _ð _ð _ñô ð ð  *Ð*;Ô<ˆÔÐÐrE   r“   c                 ó`  — | j         dk    rd S | j        �ƒ| j        |vrbd| j        ›dt          |¦  «        › d| j        ›d�}|j        j        j        j        rt          |¦  «        ‚t           	                    |¦  «         |  
                    ||¦  «         d S |                      ||¦  «         d S )Nr   z`ModelCheckpoint(monitor=z=)` could not find the monitored key in the returned metrics: z. HINT: Did you call `log(z#, value)` in the `LightningModule`?)r$   r    r�   r‹   rŒ   Úval_loopÚ_has_runr   Úwarning_cacher·   Ú_save_monitor_checkpointÚ_save_none_monitor_checkpoint)rB   rN   r“   Úms       rD   r…   z%ModelCheckpoint._save_topk_checkpoint>  sè   € ØŒ?˜aÒÐØˆFð Œ<Ð#ØŒ|Ð#5Ð5Ð5ðd°´ð dð dÝ!%Ð&8Ñ!9Ô!9ðdð dà04´ðdð dð dð ð
 Ô#Ô.Ô7Ô@ð 7Ý3°AÑ6Ô6Ð6Ý×"Ò" 1Ñ%Ô%Ð%Ø×)Ò)¨'Ð3EÑFÔFÐFÐFÐFà×.Ò.¨wÐ8JÑKÔKÐKÐKÐKrE   r­   c                 óÌ   — |                      || j        ¦  «         |j        | _        || _        |j        r,|j        D ]&}|                     t          | ¦  «        ¦  «         Œ%dS dS )zìSave the checkpoint to the given filepath.

        For manual optimization, we rely on the fact that the model's training_step method saves the model state before
        the optimizer step, so we can use that state directly.

        N)	Úsave_checkpointr&   ru   r4   r<   rY   ÚloggersÚafter_save_checkpointr
   )rB   rN   r­   Úloggers       rD   r©   z ModelCheckpoint._save_checkpointQ  s   € ð 	×Ò ¨$Ô*@ÑAÔAÐAØ'.Ô':ˆÔ$Ø&.ˆÔ#ð Ô!ð 	:Ø!œ/ð :ð :�Ø×,Ò,­U°4©[¬[Ñ9Ô9Ð9Ð9ð	:ð 	:ð:ð :rE   Úlinkpathc                 óÂ  — | j         �r=t          j                             |¦  «        t          j                             |¦  «        k    rýt          j                             |¦  «        st          j                             |¦  «        rt          j        |¦  «         n3t          j                             |¦  «        rt          j	        |¦  «         	 t          j
        t          j                             |t          j                             |¦  «        ¦  «        |¦  «         n%# t          $ r t          j        ||¦  «         Y nw xY w| j                             ¦   «          d S r^   )rY   ÚosÚpathÚabspathÚislinkÚisfileÚremoveÚisdirÚshutilÚrmtreeÚsymlinkÚrelpathÚdirnameÚOSErrorÚcopyrV   Úbarrier)rN   r­   rÅ   s      rD   Ú_link_checkpointz ModelCheckpoint._link_checkpointa  s#  € àÔ!ñ 
	0¥b¤g§o¢o°hÑ&?Ô&?Å2Ä7Ç?Â?ÐS[ÑC\ÔC\Ò&\Ð&\ÝŒw�~Š~˜hÑ'Ô'ð (­2¬7¯>ª>¸(Ñ+CÔ+Cð (Ý”	˜(Ñ#Ô#Ð#Ð#Ý”—’˜xÑ(Ô(ð (Ý”˜hÑ'Ô'Ð'ð0Ý”
�2œ7Ÿ?š?¨8µR´W·_²_ÀXÑ5NÔ5NÑOÔOÐQYÑZÔZÐZÐZøÝð 0ð 0ð 0õ ”˜H hÑ/Ô/Ð/Ð/Ð/ð0øøøð 	Ô× Ò Ñ"Ô"Ð"Ð"Ð"s   ÃAD! Ä!EÅEc                 óŽ   — ddl m} t          |j        ¦  «        p+|j        j        |j        k    p|j        p| j        |j	        k    S )Nr   )Ú	TrainerFn)
Ú pytorch_lightning.trainer.statesrØ   ÚboolÚfast_dev_runÚstateÚfnÚFITTINGÚsanity_checkingr4   ru   )rB   rN   rØ   s      rD   rv   z.ModelCheckpoint._should_skip_saving_checkpointp  sa   € Ø>Ð>Ð>Ð>Ð>Ð>õ �Ô%Ñ&Ô&ð CØŒ}Ô 9Ô#4Ò4ðCàÔ&ðCð Ô+¨wÔ/BÒBð		
rE   c                 ój   — | j         o,t          |j        ¦  «         o|j         o| j        |j        k    S r^   )r%   rÚ   rÛ   rß   r4   ru   )rB   rN   s     rD   r§   z)ModelCheckpoint._should_save_on_exceptionz  sI   € àÔ"ð DÝ˜Ô-Ñ.Ô.Ð.ðDàÔ+Ð+ðDð Ô,°Ô0CÒCð		
rE   c                 óî   — | j         �| j         S t          |dd ¦  «        �dS |j        dk    rdS t          |j        t
          ¦  «        rt          |j        ¦  «        n|j        }|dk    rdS |j        dk    S )NÚ_val_check_time_intervalFr   r   Tg      ð?)r2   rŠ   r‘   ry   rŽ   r�   r�   Úval_check_interval)rB   rN   rŽ   s      rD   r‰   z/ModelCheckpoint._should_save_on_train_epoch_end‚  s–   € ØÔ(Ð4ØÔ0Ð0õ �7Ð6¸Ñ=Ô=ÐIØ�5ð Ô*¨aÒ/Ð/Ø�5õ -7°wÔ7NÕPTÑ,UÔ,UÐr�C�Ô'Ñ(Ô(Ð(Ð[bÔ[rð 	ð ˜aÒÐØ�4ð Ô)¨SÒ0Ð0rE   c           	      óÒ  — | j         dk     rt          d| j         › d�¦  «        ‚| j        dk     rt          d| j        › d�¦  «        ‚| j        dk     rt          d| j        › d�¦  «        ‚| j        dk    }| j        dk    }| j        d u}||z   |z   dk    r(t          d	| j        › d
| j        › d| j        › d�¦  «        ‚| j        €!| j         dvrt          d| j         › d�¦  «        ‚d S d S )NéÿÿÿÿzInvalid value for save_top_k=z. Must be >= -1r   z&Invalid value for every_n_train_steps=z. Must be >= 0z!Invalid value for every_n_epochs=r   z.Combination of parameters every_n_train_steps=z, every_n_epochs=z and train_time_interval=z should be mutually exclusive.)rå   r   r   zModelCheckpoint(save_top_k=zM, monitor=None) is not a valid configuration. No quantity for top_k to track.)r$   r   rI   rJ   rK   r    )rB   Úevery_n_train_steps_triggeredÚevery_n_epochs_triggeredÚtrain_time_interval_triggereds       rD   Ú__validate_init_configurationz-ModelCheckpoint.__validate_init_configurationš  si  € ØŒ?˜RÒÐÝ+Ð,lÈDÌOÐ,lÐ,lÐ,lÑmÔmÐmØÔ$ qÒ(Ð(Ý+Øb¸Ô9RÐbÐbÐbñô ð ð Ô !Ò#Ð#Ý+Ð,tÐPTÔPdÐ,tÐ,tÐ,tÑuÔuÐuà(,Ô(AÀQÒ(FÐ%Ø#'Ô#7¸1Ò#<Ð Ø(,Ô(AÈÐ(MÐ%Ø(Ð+CÑCÐFcÑcÐfgÒgÐgÝ+ð0ÀÔAZð 0ð 0Ø"&Ô"6ð0ð 0ØQUÔQjð0ð 0ð 0ñô ð ð Œ<Ð D¤O¸:Ð$EÐ$Eå+ðB¨d¬oð Bð Bð Bñô ð ð  ÐÐ$EÐ$ErE   c                 óô   — t          |r|nd¦  «        | _        |rOt          |r|nd¦  «        r<t          j                             t          j                             |¦  «        ¦  «        }|| _        || _        d S )Nr/   )	r   rX   r   rÇ   rÈ   ÚrealpathÚ
expanduserr   r   )rB   r   r   s      rD   Ú__init_ckpt_dirzModelCheckpoint.__init_ckpt_dirµ  st   € Ý!¨WÐ"< ' '¸"Ñ=Ô=ˆŒàð 	DÕ.¸'Ð/I¨w¨wÀrÑJÔJð 	DÝ”g×&Ò&¥r¤w×'9Ò'9¸'Ñ'BÔ'BÑCÔCˆGàˆŒØ ˆŒˆˆrE   c                 óü   — t          j        t           j        ¦  «        }|df| dfdœ}||vr:t          dd                     |                     ¦   «         ¦  «        › d|› �¦  «        ‚||         \  | _        | _        d S )Nr   r~   ©r   r~   z`mode` can be z, z	 but got )r{   ÚtensorÚinfr   Újoinr   r³   r'   )rB   r'   Ú	torch_infÚ	mode_dicts       rD   Ú__init_monitor_modez#ModelCheckpoint.__init_monitor_mode¾  sƒ   € Ý”L¥¤Ñ+Ô+ˆ	Ø&¨Ð.¸¸
ÀEÐ7JÐKÐKˆ	à�yÐ Ð Ý+Ð,i¸T¿YºYÀyÇ~Â~ÑGWÔGWÑ=XÔ=XÐ,iÐ,iÐcgÐ,iÐ,iÑjÔjÐjà$-¨d¤OÑ!ˆŒ˜œ	˜	˜	rE   c                 óŠ   — |€#|€!|€d}d}t                                d¦  «         n|pd}|pd}|| _        || _        || _        d S )Nr   r   zQBoth every_n_train_steps and every_n_epochs are not set. Setting every_n_epochs=1)ÚlogÚdebugrK   rJ   rI   )rB   r)   r+   r*   s       rD   Ú__init_triggerszModelCheckpoint.__init_triggersÇ  si   € ð Ð&¨>Ð+AÐFYÐFaØˆNØ"#ÐÝ�IŠIÐiÑjÔjÐjÐjà+Ð0¨qˆNØ"5Ð":¸Ðà9LˆÔ!Ø$2ˆÔØ)<ˆÔ!Ð!Ð!rE   c                 ó   — | j         S r^   )rJ   rL   s    rD   r+   zModelCheckpoint.every_n_epochsÛ  s   € àÔ#Ð#rE   Úcurrentc                 ó8  — |€dS | j         dk    rdS t          | j        ¦  «        | j         k     }|rdS t          j        t          j        dœ| j                 } ||| j        | j                 ¦  «        }|j         	                    t          |¦  «        ¦  «        }|S )NFrå   Trï   )r$   Úlenr7   r{   ÚltÚgtr'   r8   rV   Úreduce_boolean_decisionrÚ   )rB   rN   rû   Úless_than_k_modelsÚ
monitor_opÚshould_update_best_and_saves         rD   Úcheck_monitor_top_kz#ModelCheckpoint.check_monitor_top_kß  s    € Øˆ?Ø�5àŒ?˜bÒ Ð Ø�4å  Ô!3Ñ4Ô4°t´ÒFÐØð 	Ø�4å"œX­e¬hÐ7Ð7¸¼	ÔBˆ
Ø&0 j°¸$Ô:LÈTÔMeÔ:fÑ&gÔ&gÐ#ð '.Ô&6×&NÒ&NÍtÐToÑOpÔOpÑ&qÔ&qÐ#à*Ð*rE   r¦   Úprefixc                 ó®  — |sd| j         z   dz   }t          j        d|¦  «        }t          |d„ d¬¦  «        }|D ]g}|dd …         }|r$|                     ||| j        z   dz   |z   ¦  «        }|                     |d	|› d
�¦  «        }||vrt          j        d¦  «        ||<   Œh|                     |¦  «        }|�| j          	                    ||g¦  «        }|S )Nz{epoch}z{step}z(\{.*?)[:\}]c                 ó    — t          | ¦  «        S r^   )rý   )Úxs    rD   ú<lambda>z9ModelCheckpoint._format_checkpoint_name.<locals>.<lambda>  s   € ­c°!©f¬f€ rE   T)ÚkeyÚreverser   Ú{z{0[ú]r   )
ÚCHECKPOINT_JOIN_CHARÚreÚfindallÚsortedÚreplaceÚCHECKPOINT_EQUALS_CHARr{   rð   Úformatrò   )rB   r   r¦   r  r(   ÚgroupsÚgroupÚnames           rD   Ú_format_checkpoint_namez'ModelCheckpoint._format_checkpoint_nameò  s  € ð ð 	Hà  4Ô#<Ñ<¸xÑGˆHõ ”˜O¨XÑ6Ô6ˆõ ˜Ð$4Ð$4¸dÐCÑCÔCˆàð 
	0ð 
	0ˆEØ˜˜˜”9ˆDà&ð dØ#×+Ò+¨E°4¸$Ô:UÑ3UÐX[Ñ3[Ð^bÑ3bÑcÔc�ð  ×'Ò'¨¨~°d¨~¨~¨~Ñ>Ô>ˆHà˜7Ð"Ð"Ý %¤¨Q¡¤�˜‘øØ—?’? 7Ñ+Ô+ˆàÐØÔ0×5Ò5°v¸xÐ6HÑIÔIˆHàˆrE   Úverc                 ó  — |p| j         }|                      |||| j        ¬¦  «        }|�| j                             |d|› �f¦  «        }|› | j        › �}| j        r%t          j                             | j        |¦  «        n|S )að  Generate a filename according to the defined template.

        Example::

            >>> tmpdir = os.path.dirname(__file__)
            >>> ckpt = ModelCheckpoint(dirpath=tmpdir, filename='{epoch}')
            >>> os.path.basename(ckpt.format_checkpoint_name(dict(epoch=0)))
            'epoch=0.ckpt'
            >>> ckpt = ModelCheckpoint(dirpath=tmpdir, filename='{epoch:03d}')
            >>> os.path.basename(ckpt.format_checkpoint_name(dict(epoch=5)))
            'epoch=005.ckpt'
            >>> ckpt = ModelCheckpoint(dirpath=tmpdir, filename='{epoch}-{val_loss:.2f}')
            >>> os.path.basename(ckpt.format_checkpoint_name(dict(epoch=2, val_loss=0.123456)))
            'epoch=2-val_loss=0.12.ckpt'
            >>> os.path.basename(ckpt.format_checkpoint_name(dict(epoch=2, val_loss=0.12), filename='{epoch:d}'))
            'epoch=2.ckpt'
            >>> ckpt = ModelCheckpoint(dirpath=tmpdir,
            ... filename='epoch={epoch}-validation_loss={val_loss:.2f}',
            ... auto_insert_metric_name=False)
            >>> os.path.basename(ckpt.format_checkpoint_name(dict(epoch=2, val_loss=0.123456)))
            'epoch=2-validation_loss=0.12.ckpt'
            >>> ckpt = ModelCheckpoint(dirpath=tmpdir, filename='{missing:d}')
            >>> os.path.basename(ckpt.format_checkpoint_name({}))
            'missing=0.ckpt'
            >>> ckpt = ModelCheckpoint(filename='{step}')
            >>> os.path.basename(ckpt.format_checkpoint_name(dict(step=0)))
            'step=0.ckpt'

        )r  r(   Nro   )	r   r  r(   r  rò   ÚFILE_EXTENSIONr   rÇ   rÈ   )rB   r¦   r   r  r  Ú	ckpt_names         rD   r¨   z&ModelCheckpoint.format_checkpoint_name  s–   € ðH Ð,˜tœ}ˆØ×/Ò/Ø�g fÀdÔFbð 0ñ 
ô 
ˆð ˆ?ØÔ0×5Ò5°xÀÀSÀÀÐ6KÑLÔLˆHàÐ6 Ô!4Ð6Ð6ˆ	Ø8<¼ÐS�rŒw�|Š|˜DœL¨)Ñ4Ô4Ð4È)ÐSrE   c                 óÔ  — | j         �| j         S t          |j        ¦  «        dk    rœ|j        d         j        �|j        d         j        }n|j        }|j        d         j        }|j        d         j        }t          |t          ¦  «        r|nd|› �}t          j
                             |t          |¦  «        |d¦  «        }n%t          j
                             |j        d¦  «        }|S )aê  Determines model checkpoint save directory at runtime. Reference attributes from the trainer's logger to
        determine where to save checkpoints. The path for saving weights is set in this priority:

        1.  The ``ModelCheckpoint``'s ``dirpath`` if passed in
        2.  The ``Logger``'s ``log_dir`` if the trainer has loggers
        3.  The ``Trainer``'s ``default_root_dir`` if the trainer has no loggers

        The path gets extended with subdirectory "checkpoints".

        Nr   Úversion_Úcheckpoints)r   rý   rÂ   Úsave_dirÚdefault_root_dirr  Úversionry   r¬   rÇ   rÈ   rò   )rB   rN   r   r  r"  Ú	ckpt_paths         rD   Ú__resolve_ckpt_dirz"ModelCheckpoint.__resolve_ckpt_dirE  sÒ   € ð Œ<Ð#à”<ÐåˆwŒÑÔ !Ò#Ð#ØŒ˜qÔ!Ô*Ð6Ø"œ?¨1Ô-Ô6��à"Ô3�Ø”? 1Ô%Ô*ˆDØ”o aÔ(Ô0ˆGÝ!+¨GµSÑ!9Ô!9ÐS�g�gÐ?SÈ'Ð?SÐ?SˆGÝœŸš X­s°4©y¬y¸'À=ÑQÔQˆIˆIõ œŸš WÔ%=¸}ÑMÔMˆIàÐrE   c                 ó  ‡ ‡‡— ‰                       |¦  «        }d‰ j        › d�Šdt          dt          fˆˆ fd„Š‰ j                             |¦  «        r(ˆfd„‰ j                             |d¬¦  «        D ¦   «         S t          ¦   «         S )	Nú^z	(-(\d+))?rÈ   rF   c                 ór   •— | j         ‰j        k    o&t          t          j        ‰| j        ¦  «        ¦  «        S r^   )Úsuffixr  rÚ   r  ÚmatchÚstem)rÈ   Úlast_patternrB   s    €€rD   Ú_is_lastz8ModelCheckpoint._find_last_checkpoints.<locals>._is_lasth  s0   ø€ Ø”; $Ô"5Ò5Ða½$½r¼xÈÐVZÔV_Ñ?`Ô?`Ñ:aÔ:aÐarE   c                 ó~   •— h | ]9} ‰t          |¦  «        ¦  «        ¯t          j                             |¦  «        ’Œ:S ri   )r   rÇ   rÈ   Únormpath)rm   Úpr,  s     €rD   ú	<setcomp>z9ModelCheckpoint._find_last_checkpoints.<locals>.<setcomp>l  sD   ø€ ÐkÐkÐk¨AÐYaÐYaÕbfÐghÑbiÔbiÑYjÔYjÐk•B”G×$Ò$ QÑ'Ô'ÐkÐkÐkrE   F)Údetail)rU   ÚCHECKPOINT_NAME_LASTr   rÚ   rX   ÚexistsÚlsÚset)rB   rN   r#  r,  r+  s   `  @@rD   Ú_find_last_checkpointsz&ModelCheckpoint._find_last_checkpointsc  s³   øøø€ à×+Ò+¨GÑ4Ô4ˆ	Ø@˜DÔ5Ð@Ð@Ð@ˆð	b�4ð 	b¥Dð 	bð 	bð 	bð 	bð 	bð 	bð 	bð Œ8�?Š?˜9Ñ%Ô%ð 	lØkÐkÐkÐk°´·²¸YÈu°Ñ1UÔ1UÐkÑkÔkÐkÝ‰uŒuˆrE   c                 óÒ   — | j         dk    rWt          | j        |d¬¦  «        rBt          | j                             |¦  «        ¦  «        dk    rt          d|› d�¦  «         d S d S d S d S )Nr   T)ÚstrictzCheckpoint directory z exists and is not empty.)r$   r   rX   rý   r4  r   )rB   r   s     rD   Ú__warn_if_dir_not_emptyz'ModelCheckpoint.__warn_if_dir_not_emptyo  s€   € ØŒ?˜aÒÐ¥G¨D¬H°gÀdÐ$KÑ$KÔ$KÐÕPSÐTXÔT\×T_ÒT_Ð`gÑThÔThÑPiÔPiÐlmÒPmÐPmÝÐU°7ÐUÐUÐUÑVÔVÐVÐVÐVð  ÐÐÐÐPmÐPmrE   Údel_filepathc                 óô   — |                       |¦  «        }| j        r[| j        }|                      ||¦  «        r>||k    r8|                       ||¬¦  «        }|dz  }|                      ||¦  «        r||k    °8|S )N©r  r   )r¨   r3   ÚSTARTING_VERSIONÚfile_exists)rB   r“   rN   r:  r­   Úversion_cnts         rD   Ú&_get_metric_interpolated_filepath_namez6ModelCheckpoint._get_metric_interpolated_filepath_names  s›   € ð ×.Ò.Ð/AÑBÔBˆàÔ'ð 	!ØÔ/ˆKØ×"Ò" 8¨WÑ5Ô5ð !¸(ÀlÒ:RÐ:RØ×6Ò6Ð7IÈ{Ð6Ñ[Ô[�Ø˜qÑ �ð ×"Ò" 8¨WÑ5Ô5ð !¸(ÀlÒ:RÐ:Rð ˆrE   c                 ó–  — t          |j        ¦  «        }|                     d¦  «        }t          |t          ¦  «        r|                     ¦   «         nt          j        |j        ¦  «        |d<   |                     d¦  «        }t          |t          ¦  «        r|                     ¦   «         nt          j        |j	        ¦  «        |d<   |S )NÚepochÚstep)
r   Úcallback_metricsrµ   ry   r   Úintr{   rð   r’   ru   )rB   rN   r“   rB  rC  s        rD   rw   z#ModelCheckpoint._monitor_candidates€  s­   € Ý% gÔ&>Ñ?Ô?Ðð #×&Ò& wÑ/Ô/ˆÝ5?ÀÅvÑ5NÔ5NÐ&w e§i¢i¡k¤k kÕTYÔT`ÐahÔavÑTwÔTwÐ˜7Ñ#Ø!×%Ò% fÑ-Ô-ˆÝ3=¸dÅFÑ3KÔ3KÐ%r T§X¢X¡Z¤Z ZÕQVÔQ]Ð^eÔ^qÑQrÔQrÐ˜6Ñ"Ø!Ð!rE   c                 óV  — | j         sd S |                      || j        ¦  «        }| j        rk| j        }|                      ||¦  «        rN|| j        k    rC|                      || j        |¬¦  «        }|dz  }|                      ||¦  «        r|| j        k    °C| j        |c}| _        | j         dk    r/| j        r(| j        dk    r|  	                    || j        |¦  «         n|  
                    ||¦  «         |r/|                      |||¦  «        r|                      ||¦  «         d S d S d S )Nr<  r   r#   r   )r"   r¨   r2  r3   r=  r>  r;   r<   r$   rÖ   r©   Ú_should_remove_checkpointÚ_remove_checkpoint)rB   rN   r“   r­   r?  Úpreviouss         rD   r†   z%ModelCheckpoint._save_last_checkpointŠ  sm  € ØŒ~ð 	ØˆFà×.Ò.Ð/AÀ4ÔC\Ñ]Ô]ˆàÔ'ð 	!ØÔ/ˆKØ×"Ò" 8¨WÑ5Ô5ð !¸(ÀdÔFZÒ:ZÐ:ZØ×6Ò6Ð7IÈ4ÔKdÐjuÐ6ÑvÔv�Ø˜qÑ �ð ×"Ò" 8¨WÑ5Ô5ð !¸(ÀdÔFZÒ:ZÐ:Zð
 *.Ô)=¸xÐ&ˆ�$Ô&ØŒ>˜VÒ#Ð#¨Ô(CÐ#ÈÌÐ[\ÒH\ÐH\Ø×!Ò! '¨4Ô+FÈÑQÔQÐQÐQà×!Ò! '¨8Ñ4Ô4Ð4Øð 	7˜×6Ò6°wÀÈ(ÑSÔSð 	7Ø×#Ò# G¨XÑ6Ô6Ð6Ð6Ð6ð	7ð 	7ð 	7ð 	7rE   c           
      ó2  — | j         sJ ‚|                     | j         ¦  «        }|                      ||¦  «        r|€J ‚|                      |||¦  «         d S | j        r9|d         }|d         }t          d|d›d|d›d| j         ›d| j        › �¦  «         d S d S )NrB  rC  úEpoch Údú, global step ú: z was not in top )r    rµ   r  Ú_update_best_and_saver!   r   r$   )rB   rN   r“   rû   rB  rC  s         rD   r½   z(ModelCheckpoint._save_monitor_checkpointŸ  sÔ   € ØŒ|ÐÐˆ|Ø$×(Ò(¨¬Ñ6Ô6ˆØ×#Ò# G¨WÑ5Ô5ð 	xØÐ&Ð&Ð&Ø×&Ò& w°Ð9KÑLÔLÐLÐLÐLØŒ\ð 	xØ& wÔ/ˆEØ% fÔ-ˆDÝÐv EÐvÐvÐv¸4ÐvÐvÐvÀTÄ\ÐvÐvÐeiÔetÐvÐvÑwÔwÐwÐwÐwð	xð 	xrE   c                 ó  — |                       ||| j        ¦  «        }| j        |c}| _        |                      ||¦  «         | j        dk    r1|r1|                      |||¦  «        r|                      ||¦  «         d S d S d S d S )Nr   )r@  r:   r©   r$   rG  rH  )rB   rN   r“   r­   rI  s        rD   r¾   z-ModelCheckpoint._save_none_monitor_checkpointª  s£   € Ø×>Ò>Ð?QÐSZÐ\`Ô\pÑqÔqˆà)-Ô)=¸xÐ&ˆ�$Ô&Ø×Ò˜g xÑ0Ô0Ð0àŒ?˜aÒÐ HÐ°×1OÒ1OÐPWÐYaÐckÑ1lÔ1lÐØ×#Ò# G¨XÑ6Ô6Ð6Ð6Ð6ð  ÐÐÐÐÐrE   c                 ó~  — | j         dk    rt          | j        ¦  «        dz   n| j         }d }t          | j        ¦  «        |k    r'|dk    r!| j        }| j                             |¦  «         t          |t          ¦  «        rIt          j        |¦  «        r5t          j	        t          | j        dk    rdnd¦  «        |j        ¬¦  «        }|                      |||¦  «        }|| _        || j        |<   t          | j        ¦  «        |k    rQ| j        dk    rt          nt           } || j        | j        j        ¬¦  «        | _        | j        | j                 | _        | j        dk    rt           nt          } || j        | j        j        ¬¦  «        | _        | j        | j                 | _        | j        rB|d	         }|d
         }	t-          d|d›d|	d›d| j        ›d|d›d| j        d›d|›d|› �¦  «         |                      ||¦  «         |r/|                      |||¦  «        r|                      ||¦  «         d S d S d S )Nrå   r   r   r   rñ   z-inf)Údevice)r
  rB  rC  rK  rL  rM  rN  z	 reached z0.5fz (best z), saving model to z as top )r$   rý   r7   r8   Úpopry   r   r{   Úisnanrð   Úfloatr'   rR  r@  r6   r~   r   rµ   r³   r:   r9   r!   r   r    r©   rG  rH  )
rB   rû   rN   r“   rn   r:  r­   Ú_oprB  rC  s
             rD   rO  z%ModelCheckpoint._update_best_and_save³  s²  € ð ,0¬?¸bÒ+@Ð+@�C�Ô"Ñ#Ô# aÑ'Ð'ÀdÄoˆàˆÝˆtÔ!Ñ"Ô" aÒ'Ð'¨A°ªE¨EØÔ3ˆLØÔ×"Ò" <Ñ0Ô0Ð0õ �g�vÑ&Ô&ð 	j­5¬;°wÑ+?Ô+?ð 	jÝ”l¥5°$´)¸uÒ2DÐ2D¨¨È&Ñ#QÔ#QÐZaÔZhÐiÑiÔiˆGà×>Ò>Ð?QÐSZÐ\hÑiÔiˆð %ˆÔØ'.ˆÔ˜8Ñ$åˆtÔ!Ñ"Ô" aÒ'Ð'àœ eÒ+Ð+•#�#µˆCØ'* s¨4Ô+=À4ÔCUÔCYÐ'ZÑ'ZÔ'ZˆDÔ$Ø!Ô/°Ô0HÔIˆDŒNà”Y %Ò'Ð'�cˆc­SˆØ"˜s 4Ô#5¸4Ô;MÔ;QÐRÑRÔRˆÔØ $Ô 2°4Ô3GÔ HˆÔàŒ<ð 	Ø& wÔ/ˆEØ% fÔ-ˆDÝða˜Ðað að a°Ðað að a¸$¼,ð að aÐSZÐað að aØÔ/Ð`ðað aØIQðað aØ]^ðað añô ð ð 	×Ò˜g xÑ0Ô0Ð0àð 	;˜D×:Ò:¸7ÀLÐRZÑ[Ô[ð 	;Ø×#Ò# G¨\Ñ:Ô:Ð:Ð:Ð:ð	;ð 	;ð 	;ð 	;rE   c                 ó>  — d„ | j                              ¦   «         D ¦   «         }|€.| j        sJ ‚t          j                             | j        d¦  «        }| j                             |d¦  «        5 }t          j	        ||¦  «         ddd¦  «         dS # 1 swxY w Y   dS )ztSaves the `best_k_models` dict containing the checkpoint paths with the corresponding scores to a YAML
        file.c                 ó>   — i | ]\  }}||                      ¦   «         “ŒS ri   )Úitemrl   s      rD   rp   z+ModelCheckpoint.to_yaml.<locals>.<dictcomp>à  s&   € ÐEÐEÐE¡$ ! Q�!�Q—V’V‘X”XÐEÐEÐErE   Nzbest_k_models.yamlÚw)
r7   r‚   r   rÇ   rÈ   rò   rX   ÚopenÚyamlÚdump)rB   r­   Úbest_kÚfps       rD   Úto_yamlzModelCheckpoint.to_yamlÝ  sÛ   € ð FÐE¨$Ô*<×*BÒ*BÑ*DÔ*DÐEÑEÔEˆØÐØ”<ÐÐ�<Ý”w—|’| D¤LÐ2FÑGÔGˆHØŒX�]Š]˜8 SÑ)Ô)ð 	"¨RÝŒI�f˜bÑ!Ô!Ð!ð	"ð 	"ð 	"ñ 	"ô 	"ð 	"ð 	"ð 	"ð 	"ð 	"ð 	"ð 	"øøøð 	"ð 	"ð 	"ð 	"ð 	"ð 	"s   Á/BÂBÂBc                 ó€   — |j         r| j                             |¦  «        nd}|j                             |d¬¦  «        S )z‘Checks if a file exists on rank 0 and synchronizes the result to all other ranks, preventing the internal
        state to diverge between ranks.F)Úall)rY   rX   r3  rV   r   )rB   r­   rN   Úlocal_decisions       rD   r>  zModelCheckpoint.file_existsç  sA   € ð 7>Ô6LÐW˜œŸš¨Ñ2Ô2Ð2ÐRWˆàÔ×7Ò7¸ÈEÐ7ÑRÔRÐRrE   rI  c                 óX  — ||k    rdS t          |¦  «        sdS t          |¦  «                             ¦   «         }|j        �&t          |j        ¦  «                             ¦   «         nd}|�||k    rdS | j        €J ‚t          | j        ¦  «                             ¦   «         }||j        v S )aÉ  Checks if the previous checkpoint should be deleted.

        A checkpoint won't be deleted if any of the cases apply:
        - The previous checkpoint is the same as the current checkpoint (means the old was already overwritten by new)
        - The previous checkpoint is not in the current checkpoint directory and the filesystem is local
        - The previous checkpoint is the checkpoint the Trainer resumed from and the filesystem is local

        FTN)r   r   Úabsoluter#  r   Úparents)rB   rN   rI  rû   Úresume_pathr   s         rD   rG  z)ModelCheckpoint._should_remove_checkpointï  s¶   € ð �wÒÐØ�5Ý& xÑ0Ô0ð 	Ø�4Ý˜‘>”>×*Ò*Ñ,Ô,ˆØ<CÔ<MÐ<Y•d˜7Ô,Ñ-Ô-×6Ò6Ñ8Ô8Ð8Ð_cˆØÐ" x°;Ò'>Ð'>Ø�5ØŒ|Ð'Ð'Ð'Ý�t”|Ñ$Ô$×-Ò-Ñ/Ô/ˆØ˜(Ô*Ð*Ð*rE   c                 ó:   — |j                              |¦  «         dS )z1Calls the strategy to remove the checkpoint file.N)rV   Úremove_checkpoint)rB   rN   r­   s      rD   rH  z"ModelCheckpoint._remove_checkpoint  s   € àÔ×*Ò*¨8Ñ4Ô4Ð4Ð4Ð4rE   )NNNFNr   FFr   TNNNNT)rN   rO   rP   rQ   rF   N)rF   Nr^   )NT)NNN)Ar«   Ú
__module__Ú__qualname__Ú__doc__r  r  r2  r  r=  r   r   r¬   rÚ   r	   r   rE  r   r1   Úpropertyr   rM   r\   ra   r   r   rž   r¡   r£   ÚBaseExceptionr®   r°   rz   r�   r„   r   r…   r©   ÚstaticmethodrÖ   rv   r§   r‰   rA   r?   r>   r@   r+   r  r  r¨   rU   r5  r6  rZ   r@  rw   r†   r½   r¾   rO  r`  r>  rG  rH  Ú__classcell__)rC   s   @rD   r   r   2   sÝ  ø€ € € € € ðLð Lð\ ÐØ ÐØ!ÐØ€NØÐð $(Ø"&Ø!%ØØ<@ØØ"'Ø"'ØØ(,Ø-1Ø37Ø(,Ø26Ø'+ð!.-ð .-à˜%”ð.-ð ˜3”-ð.-ð ˜#”ð	.-ð
 ð.-ð ˜E $¨°¬Ð"7Ô8Ô9ð.-ð ð.-ð  ð.-ð  ð.-ð ð.-ð "&ð.-ð & cœ]ð.-ð & iÔ0ð.-ð ! œð.-ð "*¨$¤ð.-ð  !%ð!.-ð .-ð .-ð .-ð .-ð .-ð` Øð
˜3ð 
ð 
ð 
ñ „Xñ „Xð
ð ð
˜\ð 
Ð6Jð 
ÐSVð 
Ð[_ð 
ð 
ð 
ñ „Xð
ð ð3ð 3ð 3ñ „Xð3ð ðQ@àðQ@ð (ðQ@ð ð	Q@ð
 ðQ@ð ðQ@ð 
ðQ@ð Q@ð Q@ñ „XðQ@ðf ð
Hð 
Hð 
Hñ „Xð
Hð ðHð Hð Hñ „XðHð( ð
 Lð 
Ð=Qð 
Ð^kð 
Ðptð 
ð 
ð 
ñ „Xð
ðDð Dð Dð Dð ð
˜D  c œNð 
ð 
ð 
ñ „Xð
ð ð=¨$¨s°C¨x¬.ð =¸Tð =ð =ð =ñ „Xð=ð$L¨\ð LÈtÐTWÐY_ÐT_ÔO`ð LÐeið Lð Lð Lð Lð&:¨ð :Àð :Èð :ð :ð :ð :ð  ð# ,ð #¸#ð #Èð #ÐQUð #ð #ð #ñ „\ð#ð
°lð 
Àtð 
ð 
ð 
ð 
ð
°ð 
À$ð 
ð 
ð 
ð 
ð1°|ð 1Èð 1ð 1ð 1ð 1ð0ð ð ð ð6! x°¤ð !À(È3Ä-ð !ÐTXð !ð !ð !ð !ð4¨ð 4°ð 4ð 4ð 4ð 4ð=à% cœ]ð=ð ! œð=ð & iÔ0ð	=ð
 
ð=ð =ð =ð =ð( ð$ ¨¤ð $ð $ð $ñ „Xð$ð+ð +¨<ð +À(È6ÔBRð +Ð^bð +ð +ð +ð +ð. !%Ø(,ð"ð "à˜3”-ð"ð �c˜6�kÔ"ð"ð ˜”ð	"ð
 "&ð"ð 
ð"ð "ð "ð "ðN #'Ø!Ø $ð-Tð -Tà�c˜6�kÔ"ð-Tð ˜3”-ð-Tð �cŒ]ð	-Tð
 ˜”ð-Tð 
ð-Tð -Tð -Tð -Tð^¨,ð ¸5ð ð ð ð ð<
¨lð 
¸sÀ3¼xð 
ð 
ð 
ð 
ðW¨uð W¸ð Wð Wð Wð Wð
 koðð Ø"& s¨F {Ô"3ðØ>JðØZbÐcfÔZgðà	ðð ð ð ð"¨<ð "¸DÀÀfÀÔ<Mð "ð "ð "ð "ð7¨\ð 7ÈtÐTWÐY_ÐT_ÔO`ð 7Ðeið 7ð 7ð 7ð 7ð*	x°ð 	xÐRVÐWZÐ\bÐWbÔRcð 	xÐhlð 	xð 	xð 	xð 	xð7°\ð 7ÐW[Ð\_ÐagÐ\gÔWhð 7Ðmqð 7ð 7ð 7ð 7ð(;Øð(;Ø(4ð(;ØJNÈsÐTZÈ{ÔJ[ð(;à	ð(;ð (;ð (;ð (;ðT"ð " ¨¤ð "¸4ð "ð "ð "ð "ðS Eð S°Lð SÀTð Sð Sð Sð Sð+°ð +Èð +ÐWZð +Ð_cð +ð +ð +ð +ð*5¨,ð 5À#ð 5È$ð 5ð 5ð 5ð 5ð 5ð 5ð 5ð 5rE   r   )0rl  ÚloggingrÇ   r  rÎ   r_   r¶   rÔ   r   Údatetimer   Úpathlibr   Útypingr   r   r   r	   Úweakrefr
   r{   r\  r   Útyping_extensionsr   Úpytorch_lightningÚplÚ#lightning_fabric.utilities.cloud_ior   r   r   Ú lightning_fabric.utilities.typesr   Úpytorch_lightning.callbacksr   Ú&pytorch_lightning.utilities.exceptionsr   Ú%pytorch_lightning.utilities.rank_zeror   r   r   Ú!pytorch_lightning.utilities.typesr   Ú	getLoggerr«   r÷   r¼   r   ri   rE   rD   ú<module>r€     sÔ  ððð ð €€€Ø 	€	€	€	Ø 	€	€	€	Ø €€€Ø €€€Ø €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø Ð Ð Ð Ð Ð à €€€Ø €€€Ø Ð Ð Ð Ð Ð Ø &Ð &Ð &Ð &Ð &Ð &à Ð Ð Ð Ø `Ð `Ð `Ð `Ð `Ð `Ð `Ð `Ð `Ð `Ø 2Ð 2Ð 2Ð 2Ð 2Ð 2Ø 2Ð 2Ð 2Ð 2Ð 2Ð 2Ø LÐ LÐ LÐ LÐ LÐ LØ ^Ð ^Ð ^Ð ^Ð ^Ð ^Ð ^Ð ^Ð ^Ð ^Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9à€gÔ˜Ñ!Ô!€Ø�‘”€ðT5ð T5ð T5ð T5ð T5�jñ T5ô T5ð T5ð T5ð T5rE   