§
    “©ºjhZ  ã                   óò  — d Z ddlZddlZddlmZ ddlmZ ddlmZ ddl	m
Z
mZmZmZ ddlmc mc mZ ddlmZ ddlmZmZ dd	lmZmZ ddlZdd
lmZ ddlmZ ddl m!Z! ddl"m#Z#m$Z$  ej%        e&¦  «        Z'ej(        ej)        ej*        ej+        dœZ,ej-        ej.        ej/        ej0        dœZ1e2ej3        e4f         Z5ee5         Z6eej7        ej8        ej9        fZ: G d„ de¦  «        Z; G d„ de¦  «        Z<dS )z
ModelPruning
^^^^^^^^^^^^
é    N)ÚSequence)Údeepcopy©Úpartial)ÚAnyÚCallableÚOptionalÚUnion)Úapply_to_collection)ÚTensorÚnn)Ú	TypedDictÚoverride)ÚCallback)ÚLightningModule)ÚMisconfigurationException)Úrank_zero_debugÚrank_zero_only)Úln_structuredÚl1_unstructuredÚrandom_structuredÚrandom_unstructuredc                   óJ   — e Zd ZU ej        ed<   eeee	f                  ed<   dS )Ú	_LayerRefÚdataÚnamesN)
Ú__name__Ú
__module__Ú__qualname__r   ÚModuleÚ__annotations__ÚlistÚtupleÚintÚstr© ó    ús/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/pytorch_lightning/callbacks/pruning.pyr   r   :   s6   € € € € € € Ø
Œ)€O€O�OØ��c˜3�h”Ô Ð Ð Ñ Ð Ð r'   r   c                   óL  — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 d:deeef         d	ed
ee	e                  de
deeeeegeeef         f         f         dee
eege
f         f         de
dee
eege
f         f         de
dee         dee         dede
ddfd„Zd;d	edefd„Zdededeeej        f         fd„Zedededefd„¦   «         Zdej        ddfd„Zedej        dej        deddfd „¦   «         Zd<d!„Zdeddfd"„Zdedeeef         fd#„Zdeddfd$„Zedej        dedeeef         fd%„¦   «         Zdeeef         ddfd&„Z e!	 d=d'e	eeef                  d(e	eeef                  deeef         ddfd)„¦   «         Z"e#d*d+d,e$d-eddfd.„¦   «         Z%d/eddfd0„Z&e#d*d+d,e$ddfd1„¦   «         Z'e#d*d+d,e$ddfd2„¦   «         Z(e#d*d+d,e$ddfd3„¦   «         Z)d,e$deeef         fd4„Z*e#d*d+d,e$d5eeef         ddfd6„¦   «         Z+e	 d>d,e$d	ed
e,e         defd7„¦   «         Z-ed8ede
fd9„¦   «         Z.dS )?ÚModelPruning)ÚweightÚbiasr&   NTç      à?Fr   Ú
pruning_fnÚparameters_to_pruneÚparameter_namesÚuse_global_unstructuredÚamountÚapply_pruningÚmake_pruning_permanentÚuse_lottery_ticket_hypothesisÚresample_parametersÚpruning_dimÚpruning_normÚverboseÚprune_on_train_epoch_endÚreturnc           	      óf  — || _         || _        || _        |	| _        || _        |p| j        | _        i | _        d| _        d| _	        | j        D ]%}|| j        vrt          d|› d| j        › �¦  «        ‚Œ&t          |t          ¦  «        r¯i }|                     ¦   «         }|t          vr:t          d|› dt          t                               ¦   «         ¦  «        › d�¦  «        ‚|                     d¦  «        r2|
€t          d¦  «        ‚|d	k    r|€t          d
¦  «        ‚||d<   |
|d<    | j        |fi |¤Ž}na|                      |¦  «        r|st          d¦  «        ‚n:t          dt          t                               ¦   «         ¦  «        › d|› d�¦  «        ‚|r&|j        dk    rt          d|› d|j        › d�¦  «        ‚|| _        || _        || _        t          |t0          t2          f¦  «        st5          |¦  «        st          d¦  «        ‚|| _        |dvrt          d¦  «        ‚|| _        dS )a…  Model pruning Callback, using PyTorch's prune utilities. This callback is responsible of pruning networks
        parameters during training.

        To learn more about pruning with PyTorch, please take a look at
        `this tutorial <https://pytorch.org/tutorials/intermediate/pruning_tutorial.html>`_.

        .. warning::  This is an :ref:`experimental <versioning:Experimental API>` feature.

        .. code-block:: python

            parameters_to_prune = [(model.mlp_1, "weight"), (model.mlp_2, "weight")]

            trainer = Trainer(
                callbacks=[
                    ModelPruning(
                        pruning_fn="l1_unstructured",
                        parameters_to_prune=parameters_to_prune,
                        amount=0.01,
                        use_global_unstructured=True,
                    )
                ]
            )

        When ``parameters_to_prune`` is ``None``, ``parameters_to_prune`` will contain all parameters from the model.
        The user can override ``filter_parameters_to_prune`` to filter any ``nn.Module`` to be pruned.

        Args:

            pruning_fn: Function from torch.nn.utils.prune module or your own PyTorch ``BasePruningMethod`` subclass.
                Can also be string e.g. `"l1_unstructured"`. See pytorch docs for more details.

            parameters_to_prune: List of tuples ``(nn.Module, "parameter_name_string")``.

            parameter_names: List of parameter names to be pruned from the nn.Module.
                Can either be ``"weight"`` or ``"bias"``.

            use_global_unstructured: Whether to apply pruning globally on the model.
                If ``parameters_to_prune`` is provided, global unstructured will be restricted on them.

            amount: Quantity of parameters to prune:

                - ``float``. Between 0.0 and 1.0. Represents the fraction of parameters to prune.
                - ``int``. Represents the absolute number of parameters to prune.
                - ``Callable``. For dynamic values. Will be called every epoch. Should return a value.

            apply_pruning: Whether to apply pruning.

                - ``bool``. Always apply it or not.
                - ``Callable[[epoch], bool]``. For dynamic values. Will be called every epoch.

            make_pruning_permanent: Whether to remove all reparameterization pre-hooks and apply masks
                when training ends or the model is saved.

            use_lottery_ticket_hypothesis: See `The lottery ticket hypothesis <https://arxiv.org/abs/1803.03635>`_:

                - ``bool``. Whether to apply it or not.
                - ``Callable[[epoch], bool]``. For dynamic values. Will be called every epoch.

            resample_parameters: Used with ``use_lottery_ticket_hypothesis``. If True, the model parameters will
                be resampled, otherwise, the exact original parameters will be used.

            pruning_dim: If you are using a structured pruning method you need to specify the dimension.

            pruning_norm: If you are using ``ln_structured`` you need to specify the norm.

            verbose: Verbosity level. 0 to disable, 1 to log overall sparsity, 2 to log per-layer sparsity

            prune_on_train_epoch_end: whether to apply pruning at the end of the training epoch.
                If this is ``False``, then the check runs at the end of the validation epoch.

        Raises:
            MisconfigurationException:
                If ``parameter_names`` is neither ``"weight"`` nor ``"bias"``,
                if the provided ``pruning_fn`` is not supported,
                if ``pruning_dim`` is not provided when ``"unstructured"``,
                if ``pruning_norm`` is not provided when ``"ln_structured"``,
                if ``pruning_fn`` is neither ``str`` nor :class:`torch.nn.utils.prune.BasePruningMethod`, or
                if ``amount`` is none of ``int``, ``float`` and ``Callable``.

        Nz%The provided `parameter_names` name: z
 isn't in zThe provided `pruning_fn` z2 isn't available in PyTorch's built-in functions: ú Ú_structuredzKWhen requesting `structured` pruning, the `pruning_dim` should be provided.r   zOWhen requesting `ln_structured` pruning, the `pruning_norm` should be provided.ÚnÚdimz\PyTorch `BasePruningMethod` is currently only supported with `use_global_unstructured=True`.z(`pruning_fn` is expected to be a str in z* or a PyTorch `BasePruningMethod`. Found: zI. HINT: if passing a `BasePruningMethod`, pass the class, not an instanceÚunstructuredzdOnly the "unstructured" PRUNING_TYPE is supported with `use_global_unstructured=True`. Found method z	 of type z. zO`amount` should be provided and be either an int, a float or Callable function.)r   é   é   z"`verbose` must be any of (0, 1, 2))Ú_use_global_unstructuredÚ_parameters_to_pruneÚ_use_lottery_ticket_hypothesisÚ_resample_parametersÚ_prune_on_train_epoch_endÚPARAMETER_NAMESÚ_parameter_namesÚ_global_kwargsÚ_original_layersÚ_pruning_method_namer   Ú
isinstancer%   ÚlowerÚ_PYTORCH_PRUNING_FUNCTIONSr"   ÚkeysÚendswithÚ_create_pruning_fnÚ_is_pruning_methodÚPRUNING_TYPEr.   Ú_apply_pruningÚ_make_pruning_permanentr$   ÚfloatÚcallabler2   Ú_verbose)Úselfr.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   ÚnameÚpruning_kwargss                   r(   Ú__init__zModelPruning.__init__B   s  € ðB )@ˆÔ%Ø$7ˆÔ!Ø.KˆÔ+Ø$7ˆÔ!Ø)AˆÔ&Ø /Ð G°4Ô3GˆÔØ.0ˆÔØ@DˆÔØ37ˆÔ!àÔ)ð 	ð 	ˆDØ˜4Ô/Ð/Ð/Ý/Øb¸DÐbÐbÈDÔL`ÐbÐbñô ð ð 0õ
 �j¥#Ñ&Ô&ð 	ØˆNØ#×)Ò)Ñ+Ô+ˆJØÕ!;Ð;Ð;Ý/ðW°ð Wð WÝ,0Õ1K×1PÒ1PÑ1RÔ1RÑ,SÔ,SðWð Wð Wñô ð ð ×"Ò" =Ñ1Ô1ð 4ØÐ&Ý3Øeñô ð ð  Ò0Ð0Ø#Ð+Ý7Ømñô ð ð +7�N 3Ñ'Ø(3�˜uÑ%Ø0˜Ô0°ÐNÐN¸~ÐNÐNˆJˆJØ×$Ò$ ZÑ0Ô0ð 
	Ø*ð Ý/Ørñô ð ðõ
 ,ð[½4Õ@Z×@_Ò@_Ñ@aÔ@aÑ;bÔ;bð [ð [Ø=Gð[ð [ð [ñô ð ð #ð 	 zÔ'>À.Ò'PÐ'PÝ+ðRØ!+ðRð RØ6@Ô6MðRð Rð Rñô ð ð
 %ˆŒØ+ˆÔØ'=ˆÔ$å˜6¥C­ <Ñ0Ô0ð 	µH¸VÑ4DÔ4Dð 	Ý+Øañô ð ð ˆŒà˜)Ð#Ð#Ý+Ð,PÑQÔQÐQàˆŒˆˆr'   c                 ó   — |S )zAThis function can be overridden to control which module to prune.r&   )r[   r/   s     r(   Úfilter_parameters_to_prunez'ModelPruning.filter_parameters_to_pruneë   s   € à"Ð"r'   Úkwargsc                 óà   — | j         rt          |         nt          |         }t          |¦  «        s
J d¦   «         ‚| j         r|| _        |j        | _        | j         r|S t          j        |fi |¤ŽS )a  This function takes `pruning_fn`, a function name.

        IF use_global_unstructured, pruning_fn will be resolved into its associated ``PyTorch BasePruningMethod`` ELSE,
        pruning_fn will be resolved into its function counterpart from `torch.nn.utils.prune`.

        z'Selected pruning method is not callable)	rD   Ú_PYTORCH_PRUNING_METHODrP   rY   rK   r   rM   r*   Ú_wrap_pruning_fn)r[   r.   ra   Úpruning_meths       r(   rS   zModelPruning._create_pruning_fnï   s�   € ð Ô,ð8Õ# JÔ/Ð/å+¨JÔ7ð 	õ
 ˜Ñ%Ô%ÐPÐPÐ'PÑPÔPÐ%ØÔ(ð 	)Ø"(ˆDÔð %1Ô$9ˆÔ!ØÔ(ð 	 ØÐÝÔ,¨\ÐDÐD¸VÐDÐDÐDr'   c                 ó   — t          | fi |¤ŽS ©Nr   )r.   ra   s     r(   rd   zModelPruning._wrap_pruning_fn  s   € å�zÐ,Ð, VÐ,Ð,Ð,r'   Úmodulec                 óð   — |                      ¦   «         D ]`\  }}t          |j        ¦  «        D ]F}|j        |         }t          |t          j        ¦  «        r|                     |¦  «         |j        |= ŒGŒadS )z£Removes pruning buffers from any pruned modules.

        Adapted from https://github.com/pytorch/pytorch/blob/v1.7.1/torch/nn/utils/prune.py#L1118-L1122

        N)Únamed_modulesr"   Ú_forward_pre_hooksrN   Úpytorch_pruneÚBasePruningMethodÚremove)r[   rh   Ú_ÚkÚhooks        r(   r4   z#ModelPruning.make_pruning_permanent	  sŒ   € ð  ×-Ò-Ñ/Ô/ð 	5ð 	5‰IˆAˆvÝ˜&Ô3Ñ4Ô4ð 5ð 5�ØÔ0°Ô3�Ý˜d¥MÔ$CÑDÔDð 5Ø—K’K Ñ'Ô'Ð'ØÔ1°!Ð4øð	5ð	5ð 	5r'   ÚnewÚoldr\   c                 ó:  — t          | |dz   ¦  «        rt          | |dz   ¦  «        nt          | |¦  «        }t          ||¦  «        }|�,|�*t          |t          ¦  «        rt          |t          ¦  «        sd S |j                             |j        ¦  «        |_        d S )NÚ_orig)ÚhasattrÚgetattrrN   r   r   ÚtoÚdevice)rr   rs   r\   ÚdstÚsrcs        r(   Ú_copy_paramzModelPruning._copy_param  s’   € õ /6°c¸4À'¹>Ñ.JÔ.JÐb�g�c˜4 '™>Ñ*Ô*Ð*ÕPWÐX[Ð]aÑPbÔPbˆÝ�c˜4Ñ Ô ˆØˆ;˜#˜+­Z¸½VÑ-DÔ-D˜+ÍJÐWZÕ\bÑLcÔLc˜+ØˆFØ”8—;’;˜sœzÑ*Ô*ˆŒˆˆr'   c                 ój  — | j         €J ‚| j                              ¦   «         D ]�}|d         }|d         }| j        rGt          |d¦  «        r7t	          |j        ¦  «        r#t          |¦  «        }|                     ¦   «          |D ],\  }}| j        |         \  }}|                      |||¦  «         Œ-Œ�dS )aÐ  Lottery ticket hypothesis algorithm (see page 2 of the paper):

            1. Randomly initialize a neural network :math:`f(x; \theta_0)` (where :math:`\theta_0 \sim \mathcal{D}_\theta`).
            2. Train the network for :math:`j` iterations, arriving at parameters :math:`\theta_j`.
            3. Prune :math:`p\%` of the parameters in :math:`\theta_j`, creating a mask :math:`m`.
            4. Reset the remaining parameters to their values in :math:`\theta_0`, creating the winning ticket :math:`f(x; m \odot \theta_0)`.

        This function implements the step 4.

        The ``resample_parameters`` argument can be used to reset the parameters with a new :math:`\theta_z \sim \mathcal{D}_\theta`

        Nr   r   Úreset_parameters)	rL   ÚvaluesrG   rv   rY   r~   r   rE   r|   )r[   ÚdÚcopyr   Úir\   rr   ro   s           r(   Úapply_lottery_ticket_hypothesisz,ModelPruning.apply_lottery_ticket_hypothesis  sß   € ð Ô$Ð0Ð0Ð0ØÔ&×-Ò-Ñ/Ô/ð 	2ð 	2ˆAØ�V”9ˆDØ�g”JˆEØÔ(ð (­W°TÐ;MÑ-NÔ-Nð (ÕS[Ð\`Ô\qÑSrÔSrð (Ý ‘~”~�Ø×%Ò%Ñ'Ô'Ð'Ø ð 2ð 2‘��4ØÔ2°1Ô5‘��QØ× Ò   d¨DÑ1Ô1Ð1Ð1ð2ð	2ð 	2r'   c                 óP   — | j         D ]\  }}|                      |||¬¦  «         Œd S )N)r\   r2   )rE   r.   )r[   r2   rh   r\   s       r(   Ú_apply_local_pruningz!ModelPruning._apply_local_pruning7  s=   € Ø Ô5ð 	>ð 	>‰LˆF�DØ�OŠO˜F¨°fˆOÑ=Ô=Ð=Ð=ð	>ð 	>r'   c                 óâ   ‡— || j         d<   t          t          j        | j        ¦  «        j        ¦  «        Š‰                     d¦  «         ˆfd„| j                              ¦   «         D ¦   «         S )Nr2   r[   c                 ó$   •— i | ]\  }}|‰v ¯	||“ŒS r&   r&   )Ú.0rp   ÚvÚparamss      €r(   ú
<dictcomp>z7ModelPruning._resolve_global_kwargs.<locals>.<dictcomp>?  s$   ø€ ÐLÐLÐL™˜˜AÀÀVÀÀ��1ÀÀÀr'   )rK   ÚsetÚinspectÚ	signaturer.   Ú
parametersÚdiscardÚitems)r[   r2   rŠ   s     @r(   Ú_resolve_global_kwargsz#ModelPruning._resolve_global_kwargs;  sh   ø€ Ø(.ˆÔ˜HÑ%Ý•WÔ& t¤Ñ7Ô7ÔBÑCÔCˆØ�Š�vÑÔÐØLÐLÐLÐL Ô!4×!:Ò!:Ñ!<Ô!<ÐLÑLÔLÐLr'   c                 óh   — t          j        | j        fd| j        i|                      |¦  «        ¤Ž d S )NÚpruning_method)rl   Úglobal_unstructuredrE   r.   r’   )r[   r2   s     r(   Ú_apply_global_pruningz"ModelPruning._apply_global_pruningA  sM   € ÝÔ)ØÔ%ð	
ð 	
Ø6:´oð	
ØIM×IdÒIdÐekÑIlÔIlð	
ð 	
ð 	
ð 	
ð 	
r'   c                 óÌ   — |› d�}t          | |¦  «        sdS t          | |¦  «        }|dk                         ¦   «                              ¦   «         |                     ¦   «         fS )NÚ_mask)r   rB   r   )rv   rw   ÚsumÚitemÚnumel)rh   r\   ÚattrÚmasks       r(   Ú_get_pruned_statszModelPruning._get_pruned_statsF  s`   € àˆ~ˆ~ˆ~ˆÝ�v˜tÑ$Ô$ð 	Ø�4Ý�v˜tÑ$Ô$ˆØ˜’	�ŠÑ Ô ×%Ò%Ñ'Ô'¨¯ª©¬Ð5Ð5r'   c                 ó  ‡ — ‰ j         rˆ fd„‰ j        D ¦   «         }‰ j        r‰                      |¦  «         n‰                      |¦  «         ‰ j         r-ˆ fd„‰ j        D ¦   «         }‰                      |||¬¦  «         dS dS )z+Applies pruning to ``parameters_to_prune``.c                 óB   •— g | ]\  }}‰                      ||¦  «        ‘ŒS r&   ©rž   ©rˆ   Úmr?   r[   s      €r(   ú
<listcomp>z.ModelPruning.apply_pruning.<locals>.<listcomp>Q  ó-   ø€ Ð]Ð]Ð]¹4¸1¸a˜$×0Ò0°°AÑ6Ô6Ð]Ð]Ð]r'   c                 óB   •— g | ]\  }}‰                      ||¦  «        ‘ŒS r&   r¡   r¢   s      €r(   r¤   z.ModelPruning.apply_pruning.<locals>.<listcomp>Y  r¥   r'   )r2   N)rZ   rE   rD   r–   r…   Ú_log_sparsity_stats)r[   r2   Ú
prev_statsÚ
curr_statss   `   r(   r3   zModelPruning.apply_pruningN  s´   ø€ àŒ=ð 	^Ø]Ð]Ð]Ð]À4ÔC\Ð]Ñ]Ô]ˆJàÔ(ð 	.Ø×&Ò& vÑ.Ô.Ð.Ð.à×%Ò% fÑ-Ô-Ð-àŒ=ð 	LØ]Ð]Ð]Ð]À4ÔC\Ð]Ñ]Ô]ˆJØ×$Ò$ Z°ÀFÐ$ÑKÔKÐKÐKÐKð	Lð 	Lr'   ÚprevÚcurrc                 ó  — t          d„ |D ¦   «         ¦  «        }t          d„ |D ¦   «         ¦  «        }t          d„ |D ¦   «         ¦  «        }t                               d| j        › d|› d|› d||z  d›d	|› d|› d||z  d›d
�¦  «         | j        dk    rst          | j        ¦  «        D ]`\  }\  }}	||         \  }
}||         \  }}t                               d| j        › d|›d|	› d|› d|
› d|
|z  d›d	|› d||z  d›d
�¦  «         Œ_d S d S )Nc              3   ó    K  — | ]	\  }}|V — Œ
d S rg   r&   )rˆ   ro   Útotals      r(   ú	<genexpr>z3ModelPruning._log_sparsity_stats.<locals>.<genexpr>`  s&   è è € Ð6Ð6¡X Q¨˜5Ð6Ð6Ð6Ð6Ð6Ð6r'   c              3   ó    K  — | ]	\  }}|V — Œ
d S rg   r&   ©rˆ   Úzerosro   s      r(   r¯   z3ModelPruning._log_sparsity_stats.<locals>.<genexpr>a  ó&   è è € Ð:Ð:©¨°˜uÐ:Ð:Ð:Ð:Ð:Ð:r'   c              3   ó    K  — | ]	\  }}|V — Œ
d S rg   r&   r±   s      r(   r¯   z3ModelPruning._log_sparsity_stats.<locals>.<genexpr>b  r³   r'   z	Applied `z`. Pruned: ú/z (z.2%z) -> ú)rC   z` to `ú.z` with amount=z
. Pruned: )r™   ÚlogÚinforM   rZ   Ú	enumeraterE   )r[   rª   r«   r2   Útotal_paramsÚprev_total_zerosÚcurr_total_zerosr‚   rh   r\   Úprev_mask_zerosÚprev_mask_sizeÚcurr_mask_zerosÚcurr_mask_sizes                 r(   r§   z ModelPruning._log_sparsity_stats\  s  € õ Ð6Ð6°Ð6Ñ6Ô6Ñ6Ô6ˆÝÐ:Ð:°TÐ:Ñ:Ô:Ñ:Ô:ÐÝÐ:Ð:°TÐ:Ñ:Ô:Ñ:Ô:ÐÝ�ŠðZ˜Ô1ð Zð ZØ ðZð ZØ#/ðZð ZØ3CÀlÑ3RÐ\ðZð Zà ðZð Zà#/ðZð Zà3CÀlÑ3RÐYðZð Zð Zñ	
ô 	
ð 	
ð
 Œ=˜AÒÐÝ%.¨tÔ/HÑ%IÔ%Ið ð Ñ!�‘>�F˜DØ26°q´'Ñ/� Ø26°q´'Ñ/� Ý—’ðS Ô 9ð Sð SÀð Sð SÈDð Sð SÐ`fð Sð SØ'ðSð SØ+:¸^Ñ+KÐUðSð Sà'ðSð Sà+:¸^Ñ+KÐRðSð Sð Sñô ð ð ð	 Ððð r'   Útrainerz
pl.TrainerÚ	pl_moduleÚstagec           	      ó´  — |                       || j        | j        ¬¦  «        }|                      |¦  «        | _        | j        r’i | _        t          | j        ¦  «        D ]x\  }\  }}t          |¦  «        }| j                             |t          t          |¦  «        g ¬¦  «        ¦  «         | j        |         d                              ||f¦  «         Œwd S d S )N)r0   )r   r   r   )Úsanitize_parameters_to_prunerE   rJ   r`   rF   rL   rº   ÚidÚ
setdefaultr   r   Úappend)	r[   rÂ   rÃ   rÄ   r/   r‚   rh   r\   Úid_s	            r(   ÚsetupzModelPruning.setupr  sù   € à"×?Ò?Ø�tÔ0À$ÔBWð @ñ 
ô 
Ðð %)×$CÒ$CÐDWÑ$XÔ$XˆÔ!àÔ.ð 	Fð %'ˆDÔ!Ý%.¨tÔ/HÑ%IÔ%Ið Fð FÑ!�‘>�F˜DÝ˜‘j”j�ØÔ%×0Ò0°µiÅXÈfÑEUÔEUÐ]_Ð6`Ñ6`Ô6`ÑaÔaÐaØÔ% cÔ*¨7Ô3×:Ò:¸A¸t¸9ÑEÔEÐEÐEð	Fð 	FðFð Fr'   Úcurrent_epochc                 óŽ  — t          | j        ¦  «        r|                      |¦  «        n| j        }t          | j        ¦  «        r|                      |¦  «        n| j        }|r|sd S |                      |¦  «         t          | j        ¦  «        r|                      |¦  «        r n| j        r|                      ¦   «          d S d S d S rg   )rY   rV   r2   r3   rF   rƒ   )r[   rÌ   Úpruner2   s       r(   Ú_run_pruningzModelPruning._run_pruningƒ  sè   € Ý6>¸tÔ?RÑ6SÔ6SÐl�×#Ò# MÑ2Ô2Ð2ÐY]ÔYlˆÝ/7¸¼Ñ/DÔ/DÐU�—’˜]Ñ+Ô+Ð+È$Ì+ˆØð 	˜Fð 	ØˆFØ×Ò˜6Ñ"Ô"Ð"õ ˜Ô;Ñ<Ô<ð	3Ø×/Ò/°Ñ>Ô>ð	3ð 	3ð Ô4ð	3ð
 ×0Ò0Ñ2Ô2Ð2Ð2Ð2ð	3ð 	3ð 	3ð 	3r'   c                 ój   — | j         r+t          d¦  «         |                      |j        ¦  «         d S d S )Nz3`ModelPruning.on_train_epoch_end`. Applying pruning)rH   r   rÏ   rÌ   ©r[   rÂ   rÃ   s      r(   Úon_train_epoch_endzModelPruning.on_train_epoch_end‘  sC   € àÔ)ð 	7ÝÐQÑRÔRÐRØ×Ò˜iÔ5Ñ6Ô6Ð6Ð6Ð6ð	7ð 	7r'   c                 ó|   — |j         s2| j        s-t          d¦  «         |                      |j        ¦  «         d S d S d S )Nz8`ModelPruning.on_validation_epoch_end`. Applying pruning)Úsanity_checkingrH   r   rÏ   rÌ   rÑ   s      r(   Úon_validation_epoch_endz$ModelPruning.on_validation_epoch_end—  sW   € àÔ&ð 	7¨tÔ/Mð 	7ÝÐVÑWÔWÐWØ×Ò˜iÔ5Ñ6Ô6Ð6Ð6Ð6ð	7ð 	7ð 	7ð 	7r'   c                 ó`   — | j         r&t          d¦  «         |                      |¦  «         d S d S )NzJ`ModelPruning.on_train_end`. Pruning is made permanent for this checkpoint)rW   r   r4   rÑ   s      r(   Úon_train_endzModelPruning.on_train_end�  s@   € àÔ'ð 	3ÝÐhÑiÔiÐiØ×'Ò'¨	Ñ2Ô2Ð2Ð2Ð2ð	3ð 	3r'   c                 ó>  — |                      ¦   «         }d„ |D ¦   «         }|D ]S}|                     |dz   ¦  «        }|                     |dz   ¦  «        }|                     |j        ¬¦  «        |z  ||<   ŒTdt          dt          fd„}t          |t          |¦  «        S )Nc                 ód   — h | ]-}|                      d ¦  «        ¯|                     d d¦  «        ’Œ.S )r˜   Ú )rR   Úreplace)rˆ   rp   s     r(   ú	<setcomp>zEModelPruning._make_pruning_permanent_on_state_dict.<locals>.<setcomp>§  s8   € Ð_Ð_Ð_¸È1Ï:Ê:ÐV]ÑK^ÔK^Ð_˜QŸYšY w°Ñ3Ô3Ð_Ð_Ð_r'   ru   r˜   )ÚdtypeÚtensorr;   c                 ó*   — |                       ¦   «         S rg   )Úcpu)rÞ   s    r(   Úmove_to_cpuzGModelPruning._make_pruning_permanent_on_state_dict.<locals>.move_to_cpu®  s   € à—:’:‘<”<Ðr'   )Ú
state_dictÚpoprx   rÝ   r   r   )r[   rÃ   râ   Úmap_pruned_paramsÚtensor_nameÚorigr�   rá   s           r(   Ú%_make_pruning_permanent_on_state_dictz2ModelPruning._make_pruning_permanent_on_state_dict£  s¿   € Ø×)Ò)Ñ+Ô+ˆ
ð `Ð_¸ZÐ_Ñ_Ô_ÐØ,ð 	Gð 	GˆKØ—>’> +°Ñ"7Ñ8Ô8ˆDØ—>’> +°Ñ"7Ñ8Ô8ˆDà&*§g¢g°D´J gÑ&?Ô&?À$Ñ&FˆJ�{Ñ#Ð#ð	 ¥ð 	 ­6ð 	 ð 	 ð 	 ð 	 õ # :­v°{ÑCÔCÐCr'   Ú
checkpointc                 óf   — | j         r)t          d¦  «         |                      |¦  «        |d<   d S d S )NzP`ModelPruning.on_save_checkpoint`. Pruning is made permanent for this checkpointrâ   )rW   r   rç   )r[   rÂ   rÃ   rè   s       r(   Úon_save_checkpointzModelPruning.on_save_checkpoint´  sH   € àÔ'ð 	]ÝÐnÑoÔoÐoà'+×'QÒ'QÐR[Ñ'\Ô'\ˆJ�|Ñ$Ð$Ð$ð	]ð 	]r'   c                 ó,  ‡— |pt           j        }d„ |                      ¦   «         D ¦   «         Š|sˆfd„|D ¦   «         }nÕt          |t          t
          f¦  «        rªt          |¦  «        dk    r—t          d„ |D ¦   «         ¦  «        r~t          d„ |D ¦   «         ¦  «        reg g }}|D ]D\  }}|‰vr|                     |¦  «         Œt          ||¦  «        s|                     |¦  «         ŒE|s|rt          d|› d|› �¦  «        ‚nt          d¦  «        ‚|S )	a”  This function is responsible of sanitizing ``parameters_to_prune`` and ``parameter_names``. If
        ``parameters_to_prune is None``, it will be generated with all parameters of the model.

        Raises:
            MisconfigurationException:
                If ``parameters_to_prune`` doesn't exist in the model, or
                if ``parameters_to_prune`` is neither a list nor a tuple.

        c                 ó<   — g | ]}t          |t          ¦  «        °|‘ŒS r&   )rN   Ú_MODULE_CONTAINERS)rˆ   r£   s     r(   r¤   z=ModelPruning.sanitize_parameters_to_prune.<locals>.<listcomp>Ê  s(   € ÐcÐcÐc ÅÈAÕOaÑAbÔAbÐc˜1ÐcÐcÐcr'   c           
      ó–   •— g | ]E}‰D ]@}t          ||d ¦  «        ®t          t          ||d ¦  «        t          j        ¦  «        ¯<||f‘ŒAŒFS rg   )rw   rN   r   Ú	Parameter)rˆ   Úpr£   Úcurrent_moduless      €r(   r¤   z=ModelPruning.sanitize_parameters_to_prune.<locals>.<listcomp>Í  sp   ø€ ð #ð #ð #àØ(ð#ð #ð Ý˜1˜a Ñ&Ô&Ð2µzÅ'È!ÈQÐPTÑBUÔBUÕWYÔWcÑ7dÔ7dÐ2ð �A�ð 3Ð2Ð2Ð2r'   r   c              3   ó<   K  — | ]}t          |¦  «        d k    V — ŒdS )rC   N)Úlen)rˆ   rð   s     r(   r¯   z<ModelPruning.sanitize_parameters_to_prune.<locals>.<genexpr>Ö  s,   è è € Ð=Ð= A•C˜‘F”F˜a’KÐ=Ð=Ð=Ð=Ð=Ð=r'   c              3   óz   K  — | ]6\  }}t          |t          j        ¦  «        ot          |t          ¦  «        V — Œ7d S rg   )rN   r   r    r%   )rˆ   ÚaÚbs      r(   r¯   z<ModelPruning.sanitize_parameters_to_prune.<locals>.<genexpr>×  sA   è è € ÐdÐdÉÈÈ1•J˜q¥"¤)Ñ,Ô,ÐCµ¸A½sÑ1CÔ1CÐdÐdÐdÐdÐdÐdr'   zUSome provided `parameters_to_prune` don't exist in the model. Found missing modules: z and missing parameters: zThe provided `parameters_to_prune` should either be list of tuple with 2 elements: (nn.Module, parameter_name_to_prune) or None)r*   rI   ÚmodulesrN   r"   r#   ró   ÚallrÉ   rv   r   )	rÃ   r/   r0   r�   Úmissing_modulesÚmissing_parametersrh   r\   rñ   s	           @r(   rÆ   z)ModelPruning.sanitize_parameters_to_prune»  s§  ø€ ð %ÐD­Ô(Dˆ
àcÐc i×&7Ò&7Ñ&9Ô&9ÐcÑcÔcˆà"ð 	ð#ð #ð #ð #à#ð#ñ #ô #ÐÐõ Ð*­Tµ5¨MÑ:Ô:ð	åÐ'Ñ(Ô(¨1Ò,Ð,ÝÐ=Ð=Ð)<Ð=Ñ=Ô=Ñ=Ô=ð -åÐdÐdÐPcÐdÑdÔdÑdÔdð -ð 35°bÐ/ˆOØ 3ð 4ð 4‘�˜Ø Ð0Ð0Ø#×*Ò*¨6Ñ2Ô2Ð2ØÝ˜v tÑ,Ô,ð 4Ø&×-Ò-¨dÑ3Ô3Ð3øàð Ð"4ð Ý/ðnØ/>ðnð nØYkðnð nñô ð ðõ ,ðQñô ð ð
 #Ð"r'   Úmethodc                 ób   — t          j        | ¦  «        sdS t          | t          j        ¦  «        S )NF)r�   ÚisclassÚ
issubclassrl   rm   )rû   s    r(   rT   zModelPruning._is_pruning_methodî  s,   € åŒ˜vÑ&Ô&ð 	Ø�5Ý˜&¥-Ô"AÑBÔBÐBr'   )r&   NTr-   TTTFNNr   T)r&   )r;   N)r   )r&   r&   )/r   r   r   rI   r
   r   r%   Ú_PARAM_LISTr	   r"   Úboolr$   rX   r^   r`   r   rl   rm   rS   Ústaticmethodrd   r   r    r4   r|   rƒ   r…   Údictr’   r–   r#   rž   r3   r   r§   r   r   rË   rÏ   rÒ   rÕ   r×   rç   rê   r   rÆ   rT   r&   r'   r(   r*   r*   ?   sÕ  € € € € € Ø(€Oð
 ,.Ø/3Ø(,ØHKØ<@Ø'+ØLPØ$)Ø%)Ø&*ØØ)-ðg ð g à˜( C˜-Ô(ðg ð )ðg ð " $ s¤)Ô,ð	g ð
 "&ðg ð �c˜5 (¨C¨5°%¸¸U¸
Ô2CÐ+CÔ"DÐDÔEðg ð ˜T 8¨S¨E°4¨KÔ#8Ð8Ô9ðg ð !%ðg ð (-¨T°8¸S¸EÀ4¸KÔ3HÐ-HÔ'Iðg ð "ðg ð ˜c”]ðg ð ˜s”mðg ð ðg ð #'ðg ð 
ðg ð g ð g ð g ðR#ð #¸kð #ÐS^ð #ð #ð #ð #ðE¨Sð E¸Cð EÀEÈ(ÐTaÔTsÐJsÔDtð Eð Eð Eð Eð, ð- Xð -¸ð -Àð -ð -ð -ñ „\ð-ð5¨R¬Yð 5¸4ð 5ð 5ð 5ð 5ð ð+˜œð +¨¬ð +¸#ð +À$ð +ð +ð +ñ „\ð+ð2ð 2ð 2ð 2ð0>¨5ð >°Tð >ð >ð >ð >ðM¨Uð M°t¸CÀ¸H´~ð Mð Mð Mð Mð
¨Eð 
°dð 
ð 
ð 
ð 
ð
 ð6 "¤)ð 6°3ð 6¸5ÀÀcÀ¼?ð 6ð 6ð 6ñ „\ð6ðL E¨#¨u¨*Ô$5ð L¸$ð Lð Lð Lð Lð àdeðð Ø˜˜s C˜xœÔ)ðØ15°e¸CÀ¸H´oÔ1FðØPUÐVYÐ[`ÐV`ÔPaðà	ðð ð ñ „^ðð* ðF˜\ð F°oð FÈcð FÐVZð Fð Fð Fñ „XðFð 3¨#ð 3°$ð 3ð 3ð 3ð 3ð ð7¨,ð 7À?ð 7ÐW[ð 7ð 7ð 7ñ „Xð7ð
 ð7¨|ð 7Èð 7Ð\`ð 7ð 7ð 7ñ „Xð7ð
 ð3 Lð 3¸_ð 3ÐQUð 3ð 3ð 3ñ „Xð3ð
D¸ð DÐSWÐX[Ð]`ÐX`ÔSað Dð Dð Dð Dð" ð]¨,ð ]À?ð ]Ð`dÐehÐjmÐemÔ`nð ]Ðswð ]ð ]ð ]ñ „Xð]ð àlnð0#ð 0#Ø"ð0#Ø9Dð0#Ø\dÐehÔ\ið0#à	ð0#ð 0#ð 0#ñ „\ð0#ðd ðC 3ð C¨4ð Cð Cð Cñ „\ðCð Cð Cr'   r*   )=Ú__doc__r�   ÚloggingÚcollections.abcr   r�   r   Ú	functoolsr   Útypingr   r   r	   r
   Útorch.nn.utils.pruner   ÚutilsrÎ   rl   Ú#lightning_utilities.core.apply_funcr   Útorchr   Útyping_extensionsr   r   Úpytorch_lightningÚplÚ$pytorch_lightning.callbacks.callbackr   Úpytorch_lightning.core.moduler   Ú&pytorch_lightning.utilities.exceptionsr   Ú%pytorch_lightning.utilities.rank_zeror   r   Ú	getLoggerr   r¸   r   r   r   r   rP   ÚLnStructuredÚL1UnstructuredÚRandomStructuredÚRandomUnstructuredrc   r#   r    r%   Ú_PARAM_TUPLErÿ   Ú
SequentialÚ
ModuleListÚ
ModuleDictrí   r   r*   r&   r'   r(   ú<module>r     s@  ððð ð
 €€€Ø €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ð 1Ð 1Ð 1Ð 1Ð 1Ð 1à ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø CÐ CÐ CÐ CÐ CÐ CØ Ð Ð Ð Ð Ð Ð Ð Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ð 1Ð 1à Ð Ð Ð Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø LÐ LÐ LÐ LÐ LÐ LØ QÐ QÐ QÐ QÐ QÐ QÐ QÐ Qà€gÔ˜Ñ!Ô!€ð #Ô0Ø$Ô4Ø&Ô8Ø(Ô<ð	ð Ð ð #Ô/Ø$Ô3Ø&Ô7Ø(Ô;ð	ð Ð ð �R”Y �^Ô$€Ø�|Ô$€Ø% r¤}°b´mÀRÄ]ÐSÐ ð!ð !ð !ð !ð !�	ñ !ô !ð !ð
sCð sCð sCð sCð sC�8ñ sCô sCð sCð sCð sCr'   