§
    ·¯¸j  ã                   ó„   — d Z ddlZddlZddlZddlZddlZddlZddlmZm	Z	  ej
        e¦  «        Z G d„ d¦  «        ZdS )z¡ Checkpoint Saver

Track top-n training checkpoints and maintain recovery checkpoints on specified intervals.

Hacked together by / Copyright 2020 Ross Wightman
é    Né   )Úunwrap_modelÚget_state_dictc                   ó^   — e Zd Zdddddddddedfd„Zd„ Zd	„ Zdd
„Zdd„Zdd„Z	dd„Z
d„ ZdS )ÚCheckpointSaverNÚ
checkpointÚrecoveryÚ Fé
   c                 óv  — || _         || _        || _        || _        || _        || _        g | _        d | _        d | _        d| _	        d| _
        d| _        || _        |	| _        || _        || _        d| _        |
| _        |
rt$          j        nt$          j        | _        || _        || _        | j        dk    sJ ‚d S )Nr
   Tz.pth.tarr   )ÚmodelÚtaskÚ	optimizerÚargsÚ	model_emaÚ
amp_scalerÚcheckpoint_filesÚ
best_epochÚbest_metricÚcurr_recovery_fileÚprev_recovery_fileÚcan_hardlinkÚcheckpoint_dirÚrecovery_dirÚsave_prefixÚrecovery_prefixÚ	extensionÚ
decreasingÚoperatorÚltÚgtÚcmpÚmax_historyÚ	unwrap_fn)Úselfr   r   r   r   r   Úcheckpoint_prefixr   r   r   r   r#   r$   r   s                 úk/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/utils/checkpoint_saver.pyÚ__init__zCheckpointSaver.__init__   sÌ   € ð$ ˆŒ
ØˆŒ	Ø"ˆŒØˆŒ	Ø"ˆŒØ$ˆŒð !#ˆÔØˆŒØˆÔØ"$ˆÔØ"$ˆÔØ ˆÔð -ˆÔØ(ˆÔØ,ˆÔØ.ˆÔØ#ˆŒØ$ˆŒØ",Ð=•8”;�;µ(´+ˆŒØ&ˆÔØ"ˆŒØÔ 1Ò$Ð$Ð$Ð$Ð$Ð$ó    c                 óò   — | j         rZ	 t          j                             |¦  «        rt          j        |¦  «         n%# t
          t          f$ r}d| _         Y d }~nd }~ww xY wt          j        ||¦  «         d S ©NF)r   ÚosÚpathÚexistsÚunlinkÚOSErrorÚNotImplementedErrorÚreplace©r%   ÚsrcÚdstÚes       r'   Ú_replacezCheckpointSaver._replaceD   sŠ   € ØÔð 	*ð*Ý”7—>’> #Ñ&Ô&ð #Ý”I˜c‘N”N�NøøÝÕ0Ð1ð *ð *ð *Ø$)�Ô!Ð!Ð!Ð!Ð!Ð!øøøøð*øøøå
Œ
�3˜ÑÔÐÐÐs   ‰3= ½AÁAÁAc                 ó  — | j         rp	 t          j                             |¦  «        rt          j        |¦  «         t          j        ||¦  «         d S # t          t          f$ r}d| _         Y d }~nd }~ww xY wt          j	        ||¦  «         d S r+   )
r   r,   r-   r.   r/   Úlinkr0   r1   ÚshutilÚcopy2r3   s       r'   Ú
_duplicatezCheckpointSaver._duplicateM   s    € ØÔð 	*ð*Ý”7—>’> #Ñ&Ô&ð #å”I˜c‘N”N�NÝ”˜˜SÑ!Ô!Ð!Ø�øÝÕ0Ð1ð *ð *ð *Ø$)�Ô!Ð!Ð!Ð!Ð!Ð!øøøøð*øøøåŒ�S˜#ÑÔÐÐÐs   ‰AA ÁA5Á$A0Á0A5c                 óÆ  — |t          | j        ¦  «        j                             ¦   «         | j                             ¦   «         ddœ}| j        �| j        j        |d<   | j        |d<   | j        �&| j                             ¦   «         || j        j        <   | j	        �h| 
                    | j	                             | j        ¬¦  «        ¦  «         | 
                    | j	                             d| j        ¬¦  «        ¦  «         nAt          | j        | j        ¦  «        |d<   | j        �t          | j        | j        ¦  «        |d	<   |�||d
<   t          j        ||¦  «         d S )Né   )ÚepochÚarchr   Úversionr@   r   )r$   T)Úemar$   Ú
state_dictÚstate_dict_emaÚmetric)Útyper   Ú__name__Úlowerr   rC   r   r   Ústate_dict_keyr   ÚupdateÚget_checkpoint_stater$   r   r   ÚtorchÚsave)r%   Ú	save_pathr?   rE   Ú
save_states        r'   Ú_savezCheckpointSaver._saveY   sL  € àÝ˜œÑ$Ô$Ô-×3Ò3Ñ5Ô5Øœ×2Ò2Ñ4Ô4Øð	
ð 
ˆ
ð Œ9Ð Ø!%¤¤ˆJ�vÑØ!%¤ˆJ�vÑØŒ?Ð&Ø9=¼×9SÒ9SÑ9UÔ9UˆJ�t”Ô5Ñ6ØŒ9Ð Ø×Ò˜dœi×<Ò<ÀtÄ~Ð<ÑVÔVÑWÔWÐWØ×Ò˜dœi×<Ò<ÀÐQUÔQ_Ð<Ñ`Ô`ÑaÔaÐaÐaå'5°d´jÀ$Ä.Ñ'QÔ'QˆJ�|Ñ$ØŒ~Ð)Ý/=¸d¼nÈdÌnÑ/]Ô/]�
Ð+Ñ,ØÐØ#)ˆJ�xÑ ÝŒ
�:˜yÑ)Ô)Ð)Ð)Ð)r)   r   c                 ó  — t          t          | j        ¦  «        |¦  «        }| j        |z
  }|dk     st          | j        ¦  «        |k    rd S | j        |d …         }|D ]Ž}	 t                               d                     |¦  «        ¦  «         t          j        |d         ¦  «         ŒK# t          $ r7}t           
                    d                     |¦  «        ¦  «         Y d }~Œ‡d }~ww xY w| j        d |…         | _        d S )Nr   zCleaning checkpoint: {}z(Exception '{}' while deleting checkpoint)ÚminÚlenr   r#   Ú_loggerÚdebugÚformatr,   ÚremoveÚ	ExceptionÚerror)r%   ÚtrimÚdelete_indexÚ	to_deleteÚdr6   s         r'   Ú_cleanup_checkpointsz$CheckpointSaver._cleanup_checkpointsp   s  € Ý•3�tÔ,Ñ-Ô-¨tÑ4Ô4ˆØÔ'¨$Ñ.ˆØ˜!ÒÐ�s 4Ô#8Ñ9Ô9¸\ÒIÐIØˆFØÔ)¨,¨-¨-Ô8ˆ	Øð 	Tð 	TˆAðTÝ—’Ð7×>Ò>¸qÑAÔAÑBÔBÐBÝ”	˜!˜Aœ$‘”��øÝð Tð Tð TÝ—’ÐH×OÒOÐPQÑRÔRÑSÔSÐSÐSÐSÐSÐSÐSøøøøðTøøøà $Ô 5°m°|°mÔ DˆÔÐÐs   Á!AB)Â)
C*Â3-C%Ã%C*c                 ó  — |dk    sJ ‚t           j                             | j        d| j        z   ¦  «        }t           j                             | j        d| j        z   ¦  «        }|                      |||¦  «         |                      ||¦  «         | j        r| j        d         nd }t          | j        ¦  «        | j	        k     s|�|  
                    ||d         ¦  «        �r�t          | j        ¦  «        | j	        k    r|                      d¦  «         d                     | j        t          |¦  «        g¦  «        | j        z   }t           j                             | j        |¦  «        }|                      ||¦  «         | j                             ||f¦  «         t!          | j        d„ | j         ¬¦  «        | _        d	}| j        D ]}	|d
                     |	¦  «        z  }Œt&                               |¦  «         |�s| j        �|  
                    || j        ¦  «        rQ|| _        || _        t           j                             | j        d| j        z   ¦  «        }
|                      ||
¦  «         | j        €dn| j        | j        fS )Nr   ÚtmpÚlastéÿÿÿÿr   ú-c                 ó   — | d         S )Nr   © )Úxs    r'   ú<lambda>z1CheckpointSaver.save_checkpoint.<locals>.<lambda>”   s
   € ˜a œd€ r)   )ÚkeyÚreversezCurrent checkpoints:
z {}
Ú
model_best)NN)r,   r-   Újoinr   r   rP   r7   r   rS   r#   r"   r^   r   Ústrr<   ÚappendÚsortedr   rV   rT   Úinfor   r   )r%   r?   rE   Útmp_save_pathÚlast_save_pathÚ
worst_fileÚfilenamerN   Úcheckpoints_strÚcÚbest_save_paths              r'   Úsave_checkpointzCheckpointSaver.save_checkpoint~   sZ  € Ø˜ŠzˆzˆzˆzÝœŸš TÔ%8¸%À$Ä.Ñ:PÑQÔQˆÝœŸš dÔ&9¸6ÀDÄNÑ;RÑSÔSˆØ�
Š
�= %¨Ñ0Ô0Ð0Ø�Š�m ^Ñ4Ô4Ð4à26Ô2GÐQ�TÔ*¨2Ô.Ð.ÈTˆ
å�Ô%Ñ&Ô&¨Ô)9Ò9Ð9Øˆ~Ø�xŠx˜ 
¨1¤Ñ.Ô.ñ õ �4Ô(Ñ)Ô)¨TÔ-=Ò=Ð=Ø×)Ò)¨!Ñ,Ô,Ð,Ø—x’x Ô!1µ3°u±:´:Ð >Ñ?Ô?À$Ä.ÑPˆHÝœŸš TÔ%8¸(ÑCÔCˆIØ�OŠO˜N¨IÑ6Ô6Ð6àÔ!×(Ò(¨)°VÐ)<Ñ=Ô=Ð=Ý$*ØÔ%Ø"�NØ œOÐ+ð%ñ %ô %ˆDÔ!ð 7ˆOØÔ*ð 5ð 5�Ø 7§>¢>°!Ñ#4Ô#4Ñ4��Ý�LŠL˜Ñ)Ô)Ð)àÐ! tÔ'7Ð'?À4Ç8Â8ÈFÐTXÔTdÑCeÔCeÐ'?Ø"'�”Ø#)�Ô Ý!#¤§¢¨dÔ.AÀ<ÐRVÔR`ÑC`Ñ!aÔ!a�Ø—’ °Ñ?Ô?Ð?à#Ô/Ð7ˆ|ˆ|¸dÔ>NÐPTÔP_Ð=`Ð`r)   c                 ó,  — |dk    sJ ‚t           j                             | j        d| j        z   ¦  «        }|                      ||¦  «         d                     | j        t          |¦  «        t          |¦  «        g¦  «        | j        z   }t           j                             | j        |¦  «        }|                      ||¦  «         t           j         	                    | j
        ¦  «        r—	 t                               d                     | j
        ¦  «        ¦  «         t          j        | j
        ¦  «         nJ# t          $ r=}t                               d                     || j
        ¦  «        ¦  «         Y d }~nd }~ww xY w| j        | _
        || _        d S )Nr   Úrecovery_tmprc   zCleaning recovery: {}z Exception '{}' while removing {})r,   r-   rk   r   r   rP   r   rl   r7   r.   r   rT   rU   rV   rW   rX   rY   r   )r%   r?   Ú	batch_idxrp   rs   rN   r6   s          r'   Úsave_recoveryzCheckpointSaver.save_recovery¥   si  € Ø˜ŠzˆzˆzˆzÝœŸš TÔ%6¸ÈÌÑ8WÑXÔXˆØ�
Š
�= %Ñ(Ô(Ð(à—8’8˜TÔ1µ3°u±:´:½sÀ9¹~¼~ÐNÑOÔOÐRVÔR`Ñ`ˆÝ”G—L’L Ô!2°HÑ=Ô=ˆ	Ø�Š�m YÑ/Ô/Ð/åŒ7�>Š>˜$Ô1Ñ2Ô2ð 	eðeÝ—’Ð5×<Ò<¸TÔ=TÑUÔUÑVÔVÐVÝ”	˜$Ô1Ñ2Ô2Ð2Ð2øÝð eð eð eÝ—’Ð@×GÒGÈÈ4ÔKbÑcÔcÑdÔdÐdÐdÐdÐdÐdÐdøøøøðeøøøà"&Ô"9ˆÔØ"+ˆÔÐÐs   Ã+AD7 Ä7
E>Å3E9Å9E>c                 óä   — t           j                             | j        | j        ¦  «        }t          j        |dz   | j        z   ¦  «        }t          |¦  «        }t          |¦  «        r|d         ndS )NÚ*r   r
   )	r,   r-   rk   r   r   Úglobr   rn   rS   )r%   Úrecovery_pathÚfiless      r'   Úfind_recoveryzCheckpointSaver.find_recovery·   s\   € ÝœŸš TÔ%6¸Ô8LÑMÔMˆÝ”	˜-¨#Ñ-°´Ñ>Ñ?Ô?ˆÝ�u‘”ˆÝ˜u™:œ:Ð-ˆu�QŒxˆx¨2Ð-r)   )N)r   )rG   Ú
__module__Ú__qualname__r   r(   r7   r<   rP   r^   rw   r{   r�   re   r)   r'   r   r      sÒ   € € € € € ð
 ØØØ*Ø&ØØØØØ"Øð+%ð +%ð +%ð +%ðZð ð ð
ð 
ð 
ð*ð *ð *ð *ð.Eð Eð Eð Eð%að %að %að %aðN,ð ,ð ,ð ,ð$.ð .ð .ð .ð .r)   r   )Ú__doc__r~   Úloggingr   r,   r:   rL   r   r   r   Ú	getLoggerrG   rT   r   re   r)   r'   ú<module>r‡      s¯   ððð ð €€€Ø €€€Ø €€€Ø 	€	€	€	Ø €€€à €€€à /Ð /Ð /Ð /Ð /Ð /Ð /Ð /ð ˆ'Ô
˜HÑ
%Ô
%€ðe.ð e.ð e.ð e.ð e.ñ e.ô e.ð e.ð e.ð e.r)   