§
    “©ºj›  ã                   ó    — d dl Z d dlZd dlZd dlmZmZmZmZmZ d dl	Z	d dl
mZ d dlmZ erd dlmZ  G d„ d¦  «        Z G d„ d	e¦  «        ZdS )
é    N)ÚTYPE_CHECKINGÚAnyÚLiteralÚOptionalÚUnion)Ú!_TORCHMETRICS_GREATER_EQUAL_1_0_0)Ú_PATH)ÚFabricc                   óþ  ‡ — e Zd ZdZ	 	 	 	 	 	 	 d%ded	         d
ededee         dee         dee         de	fˆ fd„Z
 ej        ¦   «         dddej        dededdf
d„¦   «         Zdej        de	fd„Zdddeddfd„Zdeeej        f         deeej        f         de	fd„Zdeeej        f         deeej        f         de	fd„Zdej        de	fd„Zd ej        ddfd!„Zdeeef         fd"„Zd#eeef         ddfd$„Zˆ xZS )&ÚSpikeDetectionaÊ  Spike Detection Callback.

    Terminates training with a ``TrainingSpikeException`` when a loss-spike was detected and
    saves the batches to skip when resuming to a file.

    We skip the current and the previous batch since it is unclear whether the previous batch
    altered the weights in a way that it causes the spike or just the current batch is corrupted somehow.

    Args:
        mode: Whether to minimize or maximize the tracked metric
        window: A running mean of metrics with ``window`` size. Serves as reference value for spikes.
        warmup: After how many batches spike-tracking should start
        atol: An absolute tolerance.  Every diff between the running mean and the current value,
            that's not an improvement and above ``atol`` will be considered a spike
        rtol: A relative tolerance. Every diff between the running mean and the current value,
            that's higher than ``rtol * running_mean`` is considered a spike
        exclude_batches_path: Where to save the file that contains the batches to exclude.
            Will default to current directory.
        finite_only: If set to ``False``, consider non-finite values like NaN, inf and -inf a spike as well.

    Úminé
   é   Nç       @TÚmode)r   ÚmaxÚwindowÚwarmupÚatolÚrtolÚexclude_batches_pathÚfinite_onlyc                 óh  •— t           rddlm} ddlm}	 nt          d¦  «        ‚t          ¦   «                              ¦   «          d| _         |	 |dd¬¦  «        |¬¦  «        | _	        d| j	        _
        d| j	        _        || _        || _        || _        || _        g | _        || _        || _        d S )	Nr   )Ú
MeanMetric)ÚRunningzJSpikeDetection requires `torchmetrics>=1.0.0` Please upgrade your version.ç        F)Údist_sync_on_stepÚsync_on_compute)r   )r   Útorchmetrics.aggregationr   Útorchmetrics.wrappersr   ÚRuntimeErrorÚsuperÚ__init__Úlast_valÚrunning_meanr   r   r   r   r   r   Úbad_batchesr   r   )Úselfr   r   r   r   r   r   r   r   r   Ú	__class__s             €úp/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/lightning_fabric/utilities/spike.pyr#   zSpikeDetection.__init__&   sÙ   ø€ õ -ð 	mØ;Ð;Ð;Ð;Ð;Ð;Ø5Ð5Ð5Ð5Ð5Ð5Ð5åÐkÑlÔlÐlÝ‰Œ×ÒÑÔÐà47ˆŒà#˜G J JÀÐX]Ð$^Ñ$^Ô$^ÐgmÐnÑnÔnˆÔà.3ˆÔÔ+Ø,1ˆÔÔ)àˆŒ	ØˆŒØˆŒ	ØˆŒ	Ø&(ˆÔØ$8ˆÔ!Ø&ˆÔÐÐó    Úfabricr
   ÚlossÚbatchÚ	batch_idxÚreturnc                 ó  — |dk    r$| j                              |j        j        ¦  «         | j        €t          j        ¦   «         | _        t          | j        ¦  «                             d¦  «        s*t
          j	         
                    | j        d¦  «        | _        t          || j        k    o|                      |¦  «        ¦  «        }|j                             ¦   «          |j                             |d¬¦  «        }|r|                      ||¦  «         dS | j        pL|j                             t          t%          j        |¦  «                             ¦   «         ¦  «        d¬¦  «        }|r|                      |¦  «         dS dS )z)Checks if we currently have a loss-spike.r   Nz.jsonzskip_batches.jsonF)ÚallT)r%   ÚtoÚstrategyÚroot_devicer   ÚosÚgetcwdÚstrÚendswithÚpathÚjoinÚboolr   Ú	_is_spikeÚbarrierÚreduce_boolean_decisionÚ_handle_spiker   ÚtorchÚisfiniter1   Ú_update_stats)r'   r+   r,   r-   r.   Úis_spikeÚis_spike_globalÚis_finite_alls           r)   Úon_train_batch_endz!SpikeDetection.on_train_batch_endF   sp  € ð ˜Š>ˆ>ØÔ× Ò  ¤Ô!<Ñ=Ô=Ð=àÔ$Ð,Ý(*¬	©¬ˆDÔ%å�4Ô,Ñ-Ô-×6Ò6°wÑ?Ô?ð 	eÝ(*¬¯ª°TÔ5NÐPcÑ(dÔ(dˆDÔ%å˜	 T¤[Ò0ÐI°T·^²^ÀDÑ5IÔ5IÑJÔJˆØŒ×ÒÑ!Ô!Ð!ð !œ/×AÒAÀ(ÐPUÐAÑVÔVˆàð 	)Ø×Ò˜v yÑ1Ô1Ð1Ð1Ð1à Ô,ð °´×0WÒ0WÝ•U”^ DÑ)Ô)×-Ò-Ñ/Ô/Ñ0Ô0°dð 1Xñ 1ô 1ˆMð ð )Ø×"Ò" 4Ñ(Ô(Ð(Ð(Ð(ð)ð )r*   c                 ó†  — t          j        ¦   «         5  t          j        d¦  «         | j                             ¦   «         }d d d ¦  «         n# 1 swxY w Y   || j        z
  }| j        rt          j        |¦  «        sdS |  	                    |¦  «        rdS |  
                    ||¦  «        o|                      ||¦  «        S )NÚignoreTF)ÚwarningsÚcatch_warningsÚsimplefilterr%   Úcomputer$   r   r@   rA   Ú
_is_betterÚ_check_atolÚ_check_rtol)r'   r,   Úrunning_valÚ	curr_diffs       r)   r<   zSpikeDetection._is_spikea   s   € õ Ô$Ñ&Ô&ð 	6ð 	6ÝÔ! (Ñ+Ô+Ð+ØÔ+×3Ò3Ñ5Ô5ˆKð	6ð 	6ð 	6ñ 	6ô 	6ð 	6ð 	6ð 	6ð 	6ð 	6ð 	6øøøð 	6ð 	6ð 	6ð 	6ð ˜4œ=Ñ(ˆ	àÔð 	¥E¤N°4Ñ$8Ô$8ð 	Ø�4à�?Š?˜9Ñ%Ô%ð 	Ø�5à×Ò  kÑ2Ô2ÐZ°t×7GÒ7GÈÈkÑ7ZÔ7ZÐZs   ”.AÁAÁAc                 óŠ  — | j                              |dz
  |g¦  «         |j        dk    rŠ| j        €J ‚t	          j        t          j                             | j        ¦  «        d¬¦  «         t          | j        d¦  «        5 }t          j
        | j         |d¬¦  «         d d d ¦  «         n# 1 swxY w Y   t          |¬¦  «        ‚)	Nr   r   T)Úexist_okÚwé   )Úindent)r.   )r&   ÚextendÚglobal_rankr   r5   Úmakedirsr9   ÚdirnameÚopenÚjsonÚdumpÚTrainingSpikeException)r'   r+   r.   Úfs       r)   r?   zSpikeDetection._handle_spikeq   sù   € ð 	Ô×Ò ¨Q¡°	Ð :Ñ;Ô;Ð;àÔ Ò"Ð"ØÔ,Ð8Ð8Ð8ÝŒK�œŸš¨Ô(AÑBÔBÈTÐRÑRÔRÐRå�dÔ/°Ñ5Ô5ð 9¸Ý”	˜$Ô*¨A°aÐ8Ñ8Ô8Ð8ð9ð 9ð 9ñ 9ô 9ð 9ð 9ð 9ð 9ð 9ð 9øøøð 9ð 9ð 9ð 9õ %¨yÐ9Ñ9Ô9Ð9s   ÂB*Â*B.Â1B.Úval_aÚval_bc                 ó~   — | j         d u p4t          t          ||z
  ¦  «        t          | j         ¦  «        k    ¦  «        S ©N)r   r;   Úabs©r'   r`   ra   s      r)   rN   zSpikeDetection._check_atol€   s5   € Ø”	˜TÐ!ÐP¥d­3¨u°u©}Ñ+=Ô+=ÅÀTÄYÁÄÒ+OÑ&PÔ&PÐPr*   c                 ó„   — | j         d u p7t          t          ||z
  ¦  «        t          | j         |z  ¦  «        k    ¦  «        S rc   )r   r;   rd   re   s      r)   rO   zSpikeDetection._check_rtolƒ   s=   € Ø”	˜TÐ!ÐX¥d­3¨u°u©}Ñ+=Ô+=ÅÀTÄYÐQVÑEVÑAWÔAWÒ+WÑ&XÔ&XÐXr*   Údiff_valc                 óð   — | j         dk    r%t          |dk                         ¦   «         ¦  «        S | j         dk    r%t          |dk                         ¦   «         ¦  «        S t          d| j         › �¦  «        ‚)Nr   r   r   r   z*Invalid mode. Has to be min or max, found )r   r;   r1   Ú
ValueError)r'   rg   s     r)   rM   zSpikeDetection._is_better†   sq   € ØŒ9˜ÒÐÝ˜ Sš×-Ò-Ñ/Ô/Ñ0Ô0Ð0ØŒ9˜ÒÐÝ˜ Qš×+Ò+Ñ-Ô-Ñ.Ô.Ð.åÐQÀdÄiÐQÐQÑRÔRÐRr*   Úvalc                 óH   — | j                              |¦  «         || _        d S rc   )r%   Úupdater$   )r'   rj   s     r)   rB   zSpikeDetection._update_statsŽ   s$   € àÔ× Ò  Ñ%Ô%Ð%ØˆŒˆˆr*   c           
      ó6  — t          | j        t          j        ¦  «        r| j                             ¦   «         n| j        | j        | j        | j        | j        | j	        | j
        | j                             ¦   «         | j        j                             ¦   «         dœ	S )N)	r$   r   r   r   r   r&   Úbad_batches_pathÚrunningÚmean)Ú
isinstancer$   r@   ÚTensorÚitemr   r   r   r   r&   r   r%   Ú
state_dictÚbase_metric)r'   s    r)   rt   zSpikeDetection.state_dict“   s…   € å0:¸4¼=Í%Ì,Ñ0WÔ0WÐj˜œ×*Ò*Ñ,Ô,Ð,Ð]aÔ]jØ”IØ”kØ”IØ”IØÔ+Ø $Ô 9ØÔ(×3Ò3Ñ5Ô5ØÔ%Ô1×<Ò<Ñ>Ô>ð

ð 

ð 
	
r*   rt   c                 ó0  — |                      d¦  «        | _        |                      d¦  «        | _        |                      d¦  «        | _        |                      d¦  «        | _        |                      d¦  «        | _        |                      d¦  «        | _        |                      d¦  «        | _        | j         	                    |                      d¦  «        ¦  «         | j
        j         	                    |                      d	¦  «        ¦  «         d S )
Nr$   r   r   r   r   r&   rn   ro   rp   )Úpopr$   r   r   r   r   r&   r   ro   Úload_state_dictr%   ru   )r'   rt   s     r)   rx   zSpikeDetection.load_state_dict    sØ   € Ø"Ÿš zÑ2Ô2ˆŒØ—N’N 6Ñ*Ô*ˆŒ	Ø —n’n XÑ.Ô.ˆŒØ—N’N 6Ñ*Ô*ˆŒ	Ø—N’N 6Ñ*Ô*ˆŒ	Ø%Ÿ>š>¨-Ñ8Ô8ˆÔØ$.§N¢NÐ3EÑ$FÔ$FˆÔ!ØŒ×$Ò$ Z§^¢^°IÑ%>Ô%>Ñ?Ô?Ð?ØÔÔ%×5Ò5°j·n²nÀVÑ6LÔ6LÑMÔMÐMÐMÐMr*   )r   r   r   Nr   NT)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Úintr   Úfloatr	   r;   r#   r@   Úno_gradrr   r   rF   r<   r?   r   rN   rO   rM   rB   Údictr7   rt   rx   Ú__classcell__©r(   s   @r)   r   r      s–  ø€ € € € € ðð ð0 ',ØØØ $Ø #Ø04Ø ð'ð 'à�lÔ#ð'ð ð'ð ð	'ð
 �uŒoð'ð �uŒoð'ð ' uœoð'ð ð'ð 'ð 'ð 'ð 'ð 'ð@ €U„]�_„_ð)¨ð )¸¼ð )Ècð )Ð^að )Ðfjð )ð )ð )ñ „_ð)ð4[˜eœlð [¨tð [ð [ð [ð [ð : Hð :¸ð :Àð :ð :ð :ð :ðQ  u¨e¬lÐ':Ô!;ð QÀEÈ%ÐQVÔQ]ÐJ]ÔD^ð QÐcgð Qð Qð Qð QðY  u¨e¬lÐ':Ô!;ð YÀEÈ%ÐQVÔQ]ÐJ]ÔD^ð YÐcgð Yð Yð Yð YðS 5¤<ð S°Dð Sð Sð Sð Sð ¤ð °$ð ð ð ð ð

˜D  c œNð 
ð 
ð 
ð 
ð	N¨$¨s°C¨x¬.ð 	N¸Tð 	Nð 	Nð 	Nð 	Nð 	Nð 	Nð 	Nð 	Nr*   r   c                   ó0   ‡ — e Zd ZdZdededefˆ fd„Zˆ xZS )r^   z,Exception to be raised with Training Spikes.r.   ÚargsÚkwargsc                 óH   •—  t          ¦   «         j        d|› �g|¢R i |¤Ž d S )Nz!Training spike detected in batch )r"   r#   )r'   r.   r„   r…   r(   s       €r)   r#   zTrainingSpikeException.__init__¯   s:   ø€ Ø�‰ŒÔÐH¸YÐHÐHÐZÈ4ÐZÐZÐZÐSYÐZÐZÐZÐZÐZr*   )ry   rz   r{   r|   r}   r   r#   r�   r‚   s   @r)   r^   r^   ¬   se   ø€ € € € € Ø6Ð6ð[ #ð [¨cð [¸Sð [ð [ð [ð [ð [ð [ð [ð [ð [ð [r*   r^   )r\   r5   rI   Útypingr   r   r   r   r   r@   Ú"lightning_fabric.utilities.importsr   Ú lightning_fabric.utilities.typesr	   Úlightning_fabric.fabricr
   r   r!   r^   © r*   r)   ú<module>rŒ      s  ðØ €€€Ø 	€	€	€	Ø €€€Ø ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?à €€€à PÐ PÐ PÐ PÐ PÐ PØ 2Ð 2Ð 2Ð 2Ð 2Ð 2àð /Ø.Ð.Ð.Ð.Ð.Ð.ðZNð ZNð ZNð ZNð ZNñ ZNô ZNð ZNðz[ð [ð [ð [ð [˜\ñ [ô [ð [ð [ð [r*   