§
    “©ºjÊG  ã                   óü   — d Z ddlmZ ddlmZmZmZmZmZm	Z	 ddl
Z
ddl
mZmZ ddlmZ ddlmZ ddlmZ ddlZdd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZmZ ddl m!Z! eeeegef         Z" G d„ de¦  «        Z#dS )zK
Stochastic Weight Averaging Callback
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
é    )Údeepcopy)ÚAnyÚCallableÚLiteralÚOptionalÚUnionÚcastN)ÚTensorÚnn)ÚLRScheduler)ÚSWALR)Úoverride)ÚCallback)ÚDeepSpeedStrategy)ÚFSDPStrategy)ÚMisconfigurationException)Úrank_zero_infoÚrank_zero_warn)ÚLRSchedulerConfigc                   ó¬  — e Zd Zdddd ej        d¦  «        fdeeee         f         deeef         ded	e	d
         de
e         de
eej        ef                  fd„Zedefd„¦   «         Zedefd„¦   «         Zedddefd„¦   «         Zedddddeddfd„¦   «         Zed0d„¦   «         Zed0d„¦   «         Zedddeddfd„¦   «         Zed0d„¦   «         Zed1d„¦   «         Zd2d „Zd3d!„Zed"dd#dd$ededdf
d%„¦   «         Zed&ed'ed(edefd)„¦   «         Z ede!eef         fd*„¦   «         Z"ed+e!eef         ddfd,„¦   «         Z#ed4d-„¦   «         Z$d.eddfd/„Z%dS )5ÚStochasticWeightAveraginggš™™™™™é?é
   ÚcosNÚcpuÚswa_lrsÚswa_epoch_startÚannealing_epochsÚannealing_strategy)r   ÚlinearÚavg_fnÚdevicec                 óJ  — d}t          |t          ¦  «        r|dk     rt          |¦  «        ‚t          |t          ¦  «        rd|cxk    rdk    sn t          |¦  «        ‚t          |t          t          f¦  «         }t          |t          ¦  «        o|dk    }	t          |t          ¦  «        ot          d„ |D ¦   «         ¦  «         }
|s|	s|
rt          d¦  «        ‚|�t          |¦  «        st          d¦  «        ‚|�3t          |t          j        t          f¦  «        st          d|› �¦  «        ‚d| _
        || _        || _        || _        || _        |p| j        | _        || _        d| _        d| _        d	| _        d| _        d| _        d| _        d
| _        i | _        |  dS )a„	  Implements the Stochastic Weight Averaging (SWA) Callback to average a model.

        Stochastic Weight Averaging was proposed in ``Averaging Weights Leads to
        Wider Optima and Better Generalization`` by Pavel Izmailov, Dmitrii
        Podoprikhin, Timur Garipov, Dmitry Vetrov and Andrew Gordon Wilson
        (UAI 2018).

        This documentation is highly inspired by PyTorch's work on SWA.
        The callback arguments follow the scheme defined in PyTorch's ``swa_utils`` package.

        For a SWA explanation, please take a look
        `here <https://pytorch.org/blog/pytorch-1.6-now-includes-stochastic-weight-averaging>`_.

        .. warning::  This is an :ref:`experimental <versioning:Experimental API>` feature.

        .. warning:: ``StochasticWeightAveraging`` is currently not supported for multiple optimizers/schedulers.

        .. warning:: ``StochasticWeightAveraging`` is currently only supported on every epoch.

        See also how to :ref:`enable it directly on the Trainer <advanced/training_tricks:Weight Averaging>`.

        Arguments:

            swa_lrs: The SWA learning rate to use:

                - ``float``. Use this value for all parameter groups of the optimizer.
                - ``List[float]``. A list values for each parameter group of the optimizer.

            swa_epoch_start: If provided as int, the procedure will start from
                the ``swa_epoch_start``-th epoch. If provided as float between 0 and 1,
                the procedure will start from ``int(swa_epoch_start * max_epochs)`` epoch

            annealing_epochs: number of epochs in the annealing phase (default: 10)

            annealing_strategy: Specifies the annealing strategy (default: "cos"):

                - ``"cos"``. For cosine annealing.
                - ``"linear"`` For linear annealing

            avg_fn: the averaging function used to update the parameters;
                the function must take in the current value of the
                :class:`AveragedModel` parameter, the current value of :attr:`model`
                parameter and the number of models already averaged; if None,
                equally weighted average is used (default: ``None``)

            device: if provided, the averaged model will be stored on the ``device``.
                When None is provided, it will infer the `device` from ``pl_module``.
                (default: ``"cpu"``)

        zBswa_epoch_start should be a >0 integer or a float between 0 and 1.é   r   c              3   óL   K  — | ]}|d k    ot          |t          ¦  «        V — Œ dS )r   N)Ú
isinstanceÚfloat)Ú.0Úlrs     ú�/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/pytorch_lightning/callbacks/stochastic_weight_avg.pyú	<genexpr>z5StochasticWeightAveraging.__init__.<locals>.<genexpr>l   s8   è è € Ð:nÐ:nÐ`b¸2Àº6Ð;[ÅjÐQSÕUZÑF[ÔF[Ð:nÐ:nÐ:nÐ:nÐ:nÐ:nó    zCThe `swa_lrs` should a positive float, or a list of positive floatsNz The `avg_fn` should be callable.z8device is expected to be a torch.device or a str. Found Féÿÿÿÿ)r%   Úintr   r&   ÚlistÚallÚcallableÚtorchr!   ÚstrÚ
n_averagedÚ_swa_epoch_startÚ_swa_lrsÚ_annealing_epochsÚ_annealing_strategyr    Ú_avg_fnÚ_deviceÚ_model_contains_batch_normÚ_average_modelÚ_initializedÚ_swa_schedulerÚ_scheduler_stateÚ_init_n_averagedÚ_latest_update_epochÚmomenta)Úselfr   r   r   r   r    r!   Úerr_msgÚ
wrong_typeÚwrong_floatÚ
wrong_lists              r)   Ú__init__z"StochasticWeightAveraging.__init__(   sÔ  € ðx WˆÝ�o¥sÑ+Ô+ð 	5°À!Ò0CÐ0CÝ+¨GÑ4Ô4Ð4Ý�o¥uÑ-Ô-ð 	5°q¸OÐ7PÐ7PÒ7PÐ7PÈqÒ7PÐ7PÐ7PÐ7PÝ+¨GÑ4Ô4Ð4å# G­eµT¨]Ñ;Ô;Ð;ˆ
Ý  ­%Ñ0Ô0ÐA°WÀ²\ˆÝ ­Ñ.Ô.ÐnµsÐ:nÐ:nÐfmÐ:nÑ:nÔ:nÑ7nÔ7nÐ3nˆ
Øð 	s˜ð 	s¨
ð 	sÝ+Ð,qÑrÔrÐràÐ¥h¨vÑ&6Ô&6ÐÝ+Ð,NÑOÔOÐOàÐ¥j°½%¼,ÍÐ9LÑ&MÔ&MÐÝ+Ð,oÐgmÐ,oÐ,oÑpÔpÐpà,0ˆŒØ /ˆÔØˆŒØ!1ˆÔØ#5ˆÔ ØÐ, ¤ˆŒØˆŒØ:>ˆÔ'Ø<@ˆÔØ!ˆÔØ59ˆÔØ04ˆÔØ !ˆÔØ$&ˆÔ!ØOQˆŒØÐÐÐr+   Úreturnc                 ój   — t          | j        t          ¦  «        sJ ‚t          | j        dz
  d¦  «        S )Nr#   r   )r%   r4   r-   Úmax©rB   s    r)   Ú	swa_startz#StochasticWeightAveraging.swa_start‡   s3   € å˜$Ô/µÑ5Ô5Ð5Ð5Ð5Ý�4Ô(¨1Ñ,¨aÑ0Ô0Ð0r+   c                 óJ   — | j         dk    rt          d¦  «        S | j         dz
  S )Nr,   Úinfr#   )Ú_max_epochsr&   rK   s    r)   Úswa_endz!StochasticWeightAveraging.swa_endŒ   s*   € àÔ˜rÒ!Ð!Ý˜‘<”<ÐØÔ !Ñ#Ð#r+   Ú	pl_moduleúpl.LightningModulec                 óX   — t          d„ |                      ¦   «         D ¦   «         ¦  «        S )Nc              3   ó^   K  — | ](}t          |t          j        j        j        ¦  «        V — Œ)d S ©N)r%   r   ÚmodulesÚ	batchnormÚ
_BatchNorm)r'   Úmodules     r)   r*   zJStochasticWeightAveraging.pl_module_contains_batch_norm.<locals>.<genexpr>”   s4   è è € ÐiÐiÈ6•:˜f¥b¤jÔ&:Ô&EÑFÔFÐiÐiÐiÐiÐiÐir+   )ÚanyrV   )rQ   s    r)   Úpl_module_contains_batch_normz7StochasticWeightAveraging.pl_module_contains_batch_norm’   s,   € åÐiÐiÐU^×UfÒUfÑUhÔUhÐiÑiÔiÑiÔiÐir+   Útrainerú
pl.TrainerÚstagec                 óŽ   — t          |j        t          t          f¦  «        rt	          d¦  «        ‚t          |¦  «        | _        d S )Nz.SWA does not currently support sharded models.)r%   Ústrategyr   r   r   r   r;   )rB   r\   rQ   r^   s       r)   ÚsetupzStochasticWeightAveraging.setup–   sE   € å�gÔ&­Õ7HÐ(IÑJÔJð 	^Ý+Ð,\Ñ]Ô]Ð]õ ' yÑ1Ô1ˆÔÐÐr+   c                 óP  — t          |j        ¦  «        dk    rt          d¦  «        ‚t          |j        ¦  «        dk    rt          d¦  «        ‚|j        €J ‚t          | j        t          ¦  «        r;|j        dk    rt          d¦  «        ‚t          |j        | j        z  ¦  «        | _        |  	                    |¦  «        | _
        |j        | _        | j
        r.|j        dk    r#|j        j        €J ‚|j        xj        dz  c_        | j        �|                      |¦  «         d S d S )Nr#   z'SWA currently works with 1 `optimizer`.z;SWA currently not supported for more than 1 `lr_scheduler`.r,   z|SWA with `swa_epoch_start` as a float is not supported when `max_epochs=-1`. Please provide `swa_epoch_start` as an integer.)ÚlenÚ
optimizersr   Úlr_scheduler_configsÚ
max_epochsr%   r4   r&   r-   r[   r:   rO   Úfit_loopr>   Ú_clear_schedulers©rB   r\   rQ   s      r)   Úon_fit_startz&StochasticWeightAveraging.on_fit_startž   sA  € åˆwÔ!Ñ"Ô" aÒ'Ð'Ý+Ð,UÑVÔVÐVåˆwÔ+Ñ,Ô,¨qÒ0Ð0Ý+Ð,iÑjÔjÐjàÔ!Ð-Ð-Ð-Ý�dÔ+­UÑ3Ô3ð 	TØÔ! RÒ'Ð'Ý/ðFñô ð õ %(¨Ô(:¸TÔ=RÑ(RÑ$SÔ$SˆDÔ!à*.×*LÒ*LÈYÑ*WÔ*WˆÔ'à"Ô-ˆÔØÔ*ð 	-¨wÔ/AÀRÒ/GÐ/GàÔ#Ô.Ð:Ð:Ð:ØÔÐ'Ô'¨1Ñ,Ð'Ô'àÔ Ð,Ø×"Ò" 7Ñ+Ô+Ð+Ð+Ð+ð -Ð,r+   c                 óB  — | j         �s`| j        |j        cxk    r| j        k    �rCn �n?d| _         | j        €J ‚| j                             | j        p|j        ¦  «        | _        |j        d         }t          | j
        t          ¦  «        r"| j
        gt          |j        ¦  «        z  | _
        t          | j
        |j        ¦  «        D ]
\  }}||d<   Œ|j        €J ‚t!          t"          t%          || j
        | j        | j        | j        dk    r|j        nd¬¦  «        ¦  «        | _        | j        � | j                             | j        ¦  «         n|j        | j        k    rt1          d¦  «         t3          | j        ¦  «        }|j        dk    sJ ‚|j        d	k    sJ ‚|j        rt|j        d         }|j        dk    s|j        d	k    rt1          d
|› �¦  «         t;          d|j        j        j         › d| j        j        j         › d�¦  «         ||j        d<   n|j         !                    |¦  «         | j"        €0tG          j$        | j%        tF          j&        |j        ¬¦  «        | _"        | j        |j        cxk    r| j        k    rXn nU|j        | j'        k    rE| j"        €J ‚| j        €J ‚|  (                    | j        || j"        | j)        ¦  «         |j        | _'        | j*        dk    rµ|j        | j        d	z   k    r¤| j        €J ‚|  +                    | j        |¦  «         |  ,                    |¦  «         |j-        xj.        d	z  c_.        d|j-        _/        |j0        | _1        t          |j-        j.        td          ¦  «        s
J d¦   «         ‚|j-        j.        |_0        d S d S d S )NTr   Ú
initial_lrr   r,   )Úswa_lrÚanneal_epochsÚanneal_strategyÚ
last_epochz—SWA is initializing after swa_start without any checkpoint data. This may be caused by loading a checkpoint from an older version of PyTorch Lightning.Úepochr#   z3SWA is currently only supported every epoch. Found zSwapping scheduler `z` for `ú`)Údtyper!   z)Iterable-style datasets are not supported)3r<   rL   Úcurrent_epochrP   r;   Útor9   r!   rd   r%   r5   r&   rc   Úparam_groupsÚziprf   r	   r   r   r6   r7   r=   r>   Úload_state_dictr   r   ÚintervalÚ	frequencyre   r   Ú	schedulerÚ	__class__Ú__name__Úappendr3   r1   Útensorr?   Úlongr@   Úupdate_parametersr8   rO   Útransfer_weightsÚreset_batch_norm_and_save_staterg   Úmax_batchesÚ_skip_backwardÚaccumulate_grad_batchesÚ_accumulate_grad_batchesr-   )rB   r\   rQ   Ú	optimizerr(   ÚgroupÚdefault_scheduler_cfgÚscheduler_cfgs           r)   Úon_train_epoch_startz.StochasticWeightAveraging.on_train_epoch_startº   s  € àÔ!ñ 6	q¨¬¸'Ô:OÐ(_Ð(_Ò(_Ð(_ÐSWÔS_Ò(_Ñ(_Ð(_Ð(_Ñ(_Ø $ˆDÔð Ô&Ð2Ð2Ð2Ø"&Ô"5×"8Ò"8¸¼Ð9YÈÔIYÑ"ZÔ"ZˆDÔàÔ*¨1Ô-ˆIÝ˜$œ-­Ñ/Ô/ð NØ!%¤ µ#°iÔ6LÑ2MÔ2MÑ M�”å  ¤°	Ô0FÑGÔGð )ð )‘	��EØ&(��lÑ#Ð#àÔ%Ð1Ð1Ð1Ý"&ÝÝØØœ=Ø"&Ô"8Ø$(Ô$<Ø59Ô5MÐQVÒ5VÐ5V˜wÔ1Ð1Ð\^ðñ ô ñ	#ô 	#ˆDÔð Ô$Ð0àÔ#×3Ò3°DÔ4IÑJÔJÐJÐJØÔ&¨$¬.Ò8Ð8õ ðmñô ð õ %6°dÔ6IÑ$JÔ$JÐ!Ø(Ô1°WÒ<Ð<Ð<Ð<Ø(Ô2°aÒ7Ð7Ð7Ð7àÔ+ð 
KØ 'Ô <¸QÔ ?�Ø Ô)¨WÒ4Ð4¸Ô8OÐSTÒ8TÐ8TÝ"Ð#hÐYfÐ#hÐ#hÑiÔiÐiÝðG¨=Ô+BÔ+LÔ+Uð Gð GØ!Ô0Ô:ÔCðGð Gð Gñô ð ð 3H�Ô,¨QÑ/Ð/àÔ,×3Ò3Ð4IÑJÔJÐJàŒÐ&Ý"'¤,¨tÔ/DÍEÌJÐ_hÔ_oÐ"pÑ"pÔ"p�”àŒN˜gÔ3ÐCÐCÒCÐC°t´|ÒCÐCÐCÐCÐCØÔ! DÔ$=Ò=Ð=à”?Ð.Ð.Ð.ØÔ&Ð2Ð2Ð2Ø×"Ò" 4Ô#6¸	À4Ä?ÐTXÔT`ÑaÔaÐaØ(/Ô(=ˆDÔ%ð Ô˜rÒ!Ð! gÔ&;¸t¼|ÈaÑ?OÒ&OÐ&OàÔ&Ð2Ð2Ð2Ø×!Ò! $Ô"5°yÑAÔAÐAð ×0Ò0°Ñ;Ô;Ð;ð
 ÔÐ(Ô(¨AÑ-Ð(Ô(Ø.2ˆGÔÔ+Ø,3Ô,KˆDÔ)Ý˜gÔ.Ô:½CÑ@Ô@ÐmÐmÐBmÑmÔmÐ@Ø.5Ô.>Ô.JˆGÔ+Ð+Ð+ð "Ð!Ð&OÐ&Or+   Úargsc                 ó   — d|j         _        d S )NF)rg   r…   )rB   r\   r�   s      r)   Úon_train_epoch_endz,StochasticWeightAveraging.on_train_epoch_end  s   € à*/ˆÔÔ'Ð'Ð'r+   c                 ó’  — | j         r{| j        dk    rp|j        dz
  | j        dz   k    rZ| j        |_        |j        xj        dz  c_        |j        j        €J ‚|j        xj        dz  c_        |  	                    ¦   «          d S |j        dz
  | j        k    s| j        dk    r&| j
        �|                      | j
        |¦  «         d S d S d S )Nr,   r#   )r:   rO   rt   rP   r‡   r†   rg   r„   rf   Úreset_momentar;   r‚   ri   s      r)   Úon_train_endz&StochasticWeightAveraging.on_train_end  sø   € ð Ô*ð 
	F¨tÔ/?À2Ò/EÐ/EÈ'ÔJ_ÐbcÑJcÐgkÔgsÐvwÑgwÒJwÐJwà.2Ô.KˆGÔ+ØÔÐ(Ô(¨AÑ-Ð(Ô(ØÔ#Ô.Ð:Ð:Ð:ØÔÐ'Ô'¨1Ñ,Ð'Ô'Ø×ÒÑ Ô Ð Ð Ð ØÔ" QÑ&¨$¬,Ò6Ð6¸$Ô:JÈbÒ:PÐ:PàÔ"Ð.Ø×%Ò% dÔ&9¸9ÑEÔEÐEÐEÐEð /Ð.ð ;QÐ:Pr+   Úsrc_pl_moduleÚdst_pl_modulec                 óø   — t          |                      ¦   «         |                     ¦   «         ¦  «        D ]D\  }}|                     ¦   «                              |                     |j        ¦  «        ¦  «         ŒEd S rU   )rw   Ú
parametersÚdetachÚcopy_ru   r!   )r“   r”   Ú	src_paramÚ	dst_params       r)   r‚   z*StochasticWeightAveraging.transfer_weights!  sx   € å$'¨×(@Ò(@Ñ(BÔ(BÀM×D\ÒD\ÑD^ÔD^Ñ$_Ô$_ð 	Eð 	EÑ ˆI�yØ×ÒÑÔ×$Ò$ Y§\¢\°)Ô2BÑ%CÔ%CÑDÔDÐDÐDð	Eð 	Er+   c                 óÎ  — i | _         |                     ¦   «         D ]È}t          |t          j        j        j        ¦  «        sŒ'|j        €J ‚t          j        |j        |j	        |j        j
        ¬¦  «        |_        |j        €J ‚t          j        |j        |j	        |j        j
        ¬¦  «        |_        |j        | j         |<   d|_        |j        €J ‚|xj        dz  c_        ŒÉdS )z_Adapted from https://github.com/pytorch/pytorch/blob/v1.7.1/torch/optim/swa_utils.py#L140-L154.N)r!   rs   r   )rA   rV   r%   r   rW   rX   Úrunning_meanr1   Ú
zeros_liker!   rs   Úrunning_varÚ	ones_likeÚmomentumÚnum_batches_tracked)rB   rQ   rY   s      r)   rƒ   z9StochasticWeightAveraging.reset_batch_norm_and_save_state&  s  € àˆŒØ×'Ò'Ñ)Ô)ð 	,ð 	,ˆFÝ˜f¥b¤jÔ&:Ô&EÑFÔFð ØØÔ&Ð2Ð2Ð2Ý"'Ô"2ØÔ#Ø Ô'ØÔ)Ô/ð#ñ #ô #ˆFÔð
 Ô%Ð1Ð1Ð1Ý!&¤ØÔ"Ø Ô'ØÔ(Ô.ð"ñ "ô "ˆFÔð
 $*¤?ˆDŒL˜Ñ Ø"ˆFŒOØÔ-Ð9Ð9Ð9ØÐ&Ô&¨!Ñ+Ð&Ô&Ð&ð%	,ð 	,r+   c                 ó>   — | j         D ]}| j         |         |_        ŒdS )z_Adapted from https://github.com/pytorch/pytorch/blob/v1.7.1/torch/optim/swa_utils.py#L164-L165.N)rA   r    )rB   Ú	bn_modules     r)   r‘   z'StochasticWeightAveraging.reset_momenta=  s/   € àœð 	9ð 	9ˆIØ!%¤¨iÔ!8ˆIÔÐð	9ð 	9r+   Úaverage_modelÚmodelr3   c                 ó‚  — t          |                      ¦   «         |                     ¦   «         ¦  «        D ]„\  }}|j        }|                     ¦   «         }|                     ¦   «                              |¦  «        }|dk    r|n ||||                     |¦  «        ¦  «        }	|                     |	¦  «         Œ…|dz  }dS )z_Adapted from https://github.com/pytorch/pytorch/blob/v1.7.1/torch/optim/swa_utils.py#L104-L112.r   r#   N)rw   r–   r!   r—   ru   r˜   )
r¤   r¥   r3   r    Úp_swaÚp_modelr!   Úp_swa_Úp_model_Úsrcs
             r)   r�   z+StochasticWeightAveraging.update_parametersB  s¸   € õ
 " -×":Ò":Ñ"<Ô"<¸e×>NÒ>NÑ>PÔ>PÑQÔQð 	ð 	‰NˆE�7Ø”\ˆFØ—\’\‘^”^ˆFØ—~’~Ñ'Ô'×*Ò*¨6Ñ2Ô2ˆHØ(¨Ašo˜o�(�(°6°6¸&À(ÈJÏMÊMÐZ`ÑLaÔLaÑ3bÔ3bˆCØ�LŠL˜ÑÔÐÐØ�a‰ˆ
ˆ
ˆ
r+   Úaveraged_model_parameterÚmodel_parameterÚnum_averagedc                 ó   — | || z
  |dz   z  z   S )z]Adapted from https://github.com/pytorch/pytorch/blob/v1.7.1/torch/optim/swa_utils.py#L95-L97.r#   © )r¬   r­   r®   s      r)   r    z StochasticWeightAveraging.avg_fnO  s!   € ð (¨?Ð=UÑ+UÐZfÐijÑZjÑ*kÑkÐkr+   c                 óÚ   — | j         €dn| j                              ¦   «         | j        | j        €d n| j                             ¦   «         | j        €d n| j                             ¦   «         dœS )Nr   )r3   Úlatest_update_epochÚscheduler_stateÚaverage_model_state)r3   Úitemr@   r=   Ú
state_dictr;   rK   s    r)   r¶   z$StochasticWeightAveraging.state_dictT  sq   € ð  $œÐ6˜!˜!¸D¼O×<PÒ<PÑ<RÔ<RØ#'Ô#<Ø'+Ô':Ð'B˜t˜tÈÔH[×HfÒHfÑHhÔHhØ+/Ô+>Ð+F 4 4ÈDÔL_×LjÒLjÑLlÔLlð	
ð 
ð 	
r+   r¶   c                 óŠ   — |d         | _         |d         | _        |d         | _        |                      |d         ¦  «         d S )Nr3   r²   r³   r´   )r?   r@   r>   Ú_load_average_model_state)rB   r¶   s     r)   rx   z)StochasticWeightAveraging.load_state_dict]  sK   € à *¨<Ô 8ˆÔØ$.Ð/DÔ$EˆÔ!Ø *Ð+<Ô =ˆÔØ×&Ò& zÐ2GÔ'HÑIÔIÐIÐIÐIr+   c                 ó~   — | j         r5t          | j         ¦  «        dk    sJ ‚| j                              ¦   «          d S d S )Nr#   )re   rc   Úclear)r\   s    r)   rh   z+StochasticWeightAveraging._clear_schedulersd  sP   € ð Ô'ð 	1Ý�wÔ3Ñ4Ô4¸Ò9Ð9Ð9Ð9ØÔ(×.Ò.Ñ0Ô0Ð0Ð0Ð0ð	1ð 	1r+   Úmodel_statec                 óL   — | j         €d S | j                              |¦  «         d S rU   )r;   rx   )rB   r»   s     r)   r¸   z3StochasticWeightAveraging._load_average_model_stateq  s-   € ØÔÐ&ØˆFØÔ×+Ò+¨KÑ8Ô8Ð8Ð8Ð8r+   )r\   r]   rQ   rR   rH   N)r“   rR   r”   rR   rH   N)rQ   rR   rH   N)rH   N)r\   r]   rH   N)&r}   Ú
__module__Ú__qualname__r1   r!   r   r&   r.   r-   r   r   Ú_AVG_FNr2   rG   ÚpropertyrL   rP   ÚstaticmethodÚboolr[   r   ra   rj   rŒ   r   r�   r’   r‚   rƒ   r‘   r
   r�   r    Údictr¶   rx   rh   r¸   r°   r+   r)   r   r   '   s­  € € € € € ð .1Ø "Ø7<Ø$(Ø5A°U´\À%Ñ5HÔ5Hð]ð ]à�u˜d 5œkÐ)Ô*ð]ð ˜s E˜zÔ*ð]ð ð	]ð
 $ OÔ4ð]ð ˜Ô!ð]ð ˜˜uœ|¨SÐ0Ô1Ô2ð]ð ]ð ]ð ]ð~ ð1˜3ð 1ð 1ð 1ñ „Xð1ð ð$˜ð $ð $ð $ñ „Xð$ð
 ðjÐ1Eð jÈ$ð jð jð jñ „\ðjð ð2˜\ð 2Ð6Jð 2ÐSVð 2Ð[_ð 2ð 2ð 2ñ „Xð2ð ð,ð ,ð ,ñ „Xð,ð6 ðQKð QKð QKñ „XðQKðf ð0¨,ð 0¸sð 0Àtð 0ð 0ð 0ñ „Xð0ð ðFð Fð Fñ „XðFð ðEð Eð Eñ „\ðEð,ð ,ð ,ð ,ð.9ð 9ð 9ð 9ð
 ð
Ø+ð
Ø4Hð
ØV\ð
Øfmð
à	ð
ð 
ð 
ñ „\ð
ð ðl¨ð lÀ&ð lÐX^ð lÐcið lð lð lñ „\ðlð ð
˜D  c œNð 
ð 
ð 
ñ „Xð
ð ðJ¨$¨s°C¨x¬.ð J¸Tð Jð Jð Jñ „XðJð ð
1ð 
1ð 
1ñ „\ð
1ð9°Sð 9¸Tð 9ð 9ð 9ð 9ð 9ð 9r+   r   )$Ú__doc__Úcopyr   Útypingr   r   r   r   r   r	   r1   r
   r   Útorch.optim.lr_schedulerr   Útorch.optim.swa_utilsr   Útyping_extensionsr   Úpytorch_lightningÚplÚ$pytorch_lightning.callbacks.callbackr   Úpytorch_lightning.strategiesr   Ú!pytorch_lightning.strategies.fsdpr   Ú&pytorch_lightning.utilities.exceptionsr   Ú%pytorch_lightning.utilities.rank_zeror   r   Ú!pytorch_lightning.utilities.typesr   r¿   r   r°   r+   r)   ú<module>rÒ      s|  ððð ð
 Ð Ð Ð Ð Ð Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø 'Ð 'Ð 'Ð 'Ð 'Ð 'Ø &Ð &Ð &Ð &Ð &Ð &à Ð Ð Ð Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø :Ð :Ð :Ð :Ð :Ð :Ø :Ð :Ð :Ð :Ð :Ð :Ø LÐ LÐ LÐ LÐ LÐ LØ PÐ PÐ PÐ PÐ PÐ PÐ PÐ PØ ?Ð ?Ð ?Ð ?Ð ?Ð ?à
�F˜F FÐ+¨VÐ3Ô
4€ðM9ð M9ð M9ð M9ð M9 ñ M9ô M9ð M9ð M9ð M9r+   