§
    ·¯¸j	u  ã                   óš  — d Z ddlZddlmZmZmZmZ ddlZddlm	Z	 ddl
m	c mZ ddlmZmZ ddlmZ ddlmZ  ej        e¦  «        Z G d„ d	e	j        ¦  «        Zd
eee	j        ef         de	j        dee         deej                 deej                 defd„Z G d„ de¦  «        Z G d„ de	j        ¦  «        Z G d„ de¦  «        ZdS )z5Knowledge distillation training tasks and components.é    N)ÚDictÚOptionalÚTupleÚUnion)Úcreate_modelÚgroup_parameters)Úunwrap_modelé   )ÚTrainingTaskc                   ó†  ‡ — e Zd ZdZ	 	 	 	 	 ddeeej        f         dee	         de	dee         dee
j                 d	ee
j                 fˆ fd
„Zde
j        de
j        fd„Z	 	 	 	 ddedee         dededd f
d„Z	 dde
j        dede
j        fd„Z	 	 dde
j        dee
j                 dee
j                 de
j        fd„Zˆ xZS )ÚDistillationTeacherae  Wrapper for a teacher model used in knowledge distillation.

    Creates and manages a pre-trained teacher model for knowledge distillation,
    handling model creation and normalization differences between teacher and student.

    Can be created from:
    - A model name string (creates the model internally with pretrained weights)
    - An existing nn.Module (wraps it with the necessary interface)

    Args:
        model_name_or_module: Either a model name string or an nn.Module
        num_classes: Number of output classes (required if model_name_or_module is a string)
        in_chans: Number of input channels (used if model_name_or_module is a string)
        pretrained_path: Optional path to pretrained weights (used if model_name_or_module is a string)
        device: Device to place the model on
        dtype: Model dtype (uses float32 if None)
    Né   Úmodel_name_or_moduleÚnum_classesÚin_chansÚpretrained_pathÚdeviceÚdtypec           	      ó–  •— t          ¦   «                              ¦   «          t          |t          ¦  «        rKt                               d|› d�¦  «         ddi}|rt          ||¬¦  «        |d<   t          d|||||dœ|¤Ž}nAt          |t          j	        ¦  «        r|}n$t          dt          |¦  «        j        › �¦  «        ‚|                     ¦   «          || _        t          |¦  «        }	t!          |	d	¦  «        r7|	j                             d
d¦  «        }
|	j                             dd¦  «        }nd}
d}t'          j        |
||¬¦  «                             dddd¦  «        }t'          j        |||¬¦  «                             dddd¦  «        }|                      d|d¬¦  «         |                      d|d¬¦  «         d | _        d S )NzCreating KD teacher model: 'ú'Ú
pretrainedT)Úfiler   Úpretrained_cfg_overlay)Ú
model_namer   r   r   r   z8model_name_or_module must be a string or nn.Module, got Úpretrained_cfgÚmean)g
×£p=
ß?gÉv¾Ÿ/Ý?g–C‹lçûÙ?Ústd)gZd;ßOÍ?gyé&1¬Ì?gÍÌÌÌÌÌÌ?©r   r   r
   éÿÿÿÿÚmean_kdF©Ú
persistentÚstd_kd© )ÚsuperÚ__init__Ú
isinstanceÚstrÚ_loggerÚinfoÚdictr   ÚnnÚModuleÚ	TypeErrorÚtypeÚ__name__ÚevalÚmodelr	   Úhasattrr   ÚgetÚtorchÚtensorÚviewÚregister_bufferÚ_compiled_forward_features)Úselfr   r   r   r   r   r   Úpretrained_kwargsr2   Úmodel_unwrappedr   r   r    r#   Ú	__class__s                 €úf/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/task/distillation.pyr&   zDistillationTeacher.__init__%   s  ø€ õ 	‰Œ×ÒÑÔÐåÐ*­CÑ0Ô0ð 	Ý�LŠLÐOÐ8LÐOÐOÐOÑPÔPÐPà!-¨tÐ 4ÐØð Ý>BØ(Ø +ð?ñ ?ô ?Ð!Ð":Ñ;õ
 !ð Ø/Ø'Ø!ØØðð ð $ðð ˆEˆEõ Ð,­b¬iÑ8Ô8ð 	Ø(ˆEˆEåØpÍ4ÐPdÑKeÔKeÔKnÐpÐpñô ð ð 	�
Š
‰ŒˆØˆŒ
õ ' uÑ-Ô-ˆÝ�?Ð$4Ñ5Ô5ð 	(Ø"Ô1×5Ò5°fÐ>SÑTÔTˆDØ!Ô0×4Ò4°UÐ<QÑRÔRˆCˆCà(ˆDØ'ˆCå”,˜t¨F¸%Ð@Ñ@Ô@×EÒEÀaÈÈQÐPQÑRÔRˆÝ”˜c¨&¸Ð>Ñ>Ô>×CÒCÀAÀrÈ1ÈaÑPÔPˆØ×Ò˜Y¨¸EÐÑBÔBÐBØ×Ò˜X v¸%ÐÑ@Ô@Ð@Ø*.ˆÔ'Ð'Ð'ó    ÚinputÚreturnc                 ó  — t          | j        d¦  «        rt          | j        d¦  «        s"t          d| j        j        j        › d�¦  «        ‚| j                             |¦  «        }| j                             |d¬¦  «        S )NÚforward_featuresÚforward_headzModel zi does not support feature extraction. Ensure the model has 'forward_features' and 'forward_head' methods.T©Ú
pre_logits)r3   r2   Ú
ValueErrorr=   r0   rC   rD   )r:   r@   Úfeature_maps      r>   Ú_forward_featuresz%DistillationTeacher._forward_features[   s‘   € Ý�t”zÐ#5Ñ6Ô6ð 	½gÀdÄjÐR`Ñ>aÔ>að 	ÝðV˜œÔ-Ô6ð Vð Vð Vñô ð ð ”j×1Ò1°%Ñ8Ô8ˆØŒz×&Ò& {¸tÐ&ÑDÔDÐDr?   ÚinductorTFÚbackendÚmodeÚcompile_modelÚcompile_featuresc                 óŠ   — |rt          j        | j        f||dœ|¤Ž| _        |rt          j        | j        f||dœ|¤Ž| _        | S )z;Compile teacher inference paths used by distillation tasks.©rK   rL   )r5   Úcompiler2   rI   r9   )r:   rK   rL   rM   rN   Úcompile_kwargss         r>   rQ   zDistillationTeacher.compiled   ss   € ð ð 	aÝœ t¤zÐ`¸7ÈÐ`Ð`ÐQ_Ð`Ð`ˆDŒJØð 	Ý.3¬mØÔ&ð/àØð/ð /ð !ð	/ð /ˆDÔ+ð ˆr?   Úreturn_featuresc                 ó’   — |r1| j         �|                       |¦  «        S |                      |¦  «        S |                      |¦  «        S )aG  Forward pass through teacher model.

        Args:
            input: Input tensor (should already be normalized for teacher)
            return_features: Whether to return pooled pre-logits features instead of logits

        Returns:
            Logits or pooled pre-logits features depending on return_features flag
        )r9   rI   r2   )r:   r@   rS   s      r>   ÚforwardzDistillationTeacher.forwardx   sO   € ð ð 	%ØÔ.Ð:Ø×6Ò6°uÑ=Ô=Ð=Ø×)Ò)¨%Ñ0Ô0Ð0à—:’:˜eÑ$Ô$Ð$r?   Ústudent_meanÚstudent_stdc                 óª   — |�|€|S t          j        || j        ¦  «        rt          j        || j        ¦  «        r|S ||z  |z   | j        z
  | j        z  S )ao  Normalize input to match teacher's expected normalization.

        Args:
            input: Input tensor (already normalized for student)
            student_mean: Student normalization mean buffer [1, 3, 1, 1]
            student_std: Student normalization std buffer [1, 3, 1, 1]

        Returns:
            Input tensor normalized for the teacher model
        )r5   Úequalr    r#   )r:   r@   rV   rW   s       r>   Únormalize_inputz#DistillationTeacher.normalize_input�   sd   € ð  Ð ;Ð#6ØˆLÝŒ;�| T¤\Ñ2Ô2ð 	µu´{À;ÐPTÔP[Ñ7\Ô7\ð 	ØˆLØ˜Ñ# lÑ2°T´\ÑAÀTÄ[ÑPÐPr?   )Nr   NNN)rJ   NTF©F)NN)r0   Ú
__module__Ú__qualname__Ú__doc__r   r(   r,   r-   r   Úintr5   r   r   r&   ÚTensorrI   ÚboolrQ   rU   rZ   Ú__classcell__©r=   s   @r>   r   r      sä  ø€ € € € € ðð ð* *.ØØ-1Ø-1Ø+/ð4/ð 4/à"'¨¨R¬Y¨Ô"7ð4/ð " #œð4/ð ð	4/ð
 & cœ]ð4/ð ˜Uœ\Ô*ð4/ð ˜EœKÔ(ð4/ð 4/ð 4/ð 4/ð 4/ð 4/ðlE u¤|ð E¸¼ð Eð Eð Eð Eð &Ø"&Ø"&Ø%*ðð àðð ˜3”-ðð  ð	ð
 #ðð 
ðð ð ð ð. %*ð%ð %à”<ð%ð "ð%ð 
Œð	%ð %ð %ð %ð0 48Ø26ð	Qð Qà”<ðQð # 5¤<Ô0ðQð " %¤,Ô/ð	Qð
 
ŒðQð Qð Qð Qð Qð Qð Qð Qr?   r   ÚteacherÚstudent_modelr   r   r   rA   c                 ó”   — t          | t          ¦  «        r| S t          |¦  «        }|j        }|j        }t          | |||||¬¦  «        S )a’  Resolve teacher input to a DistillationTeacher instance.

    Args:
        teacher: Model name string, nn.Module, or DistillationTeacher
        student_model: Student model to infer num_classes/in_chans from
        pretrained_path: Optional path to teacher pretrained weights
        device: Device for teacher
        dtype: Dtype for teacher

    Returns:
        DistillationTeacher instance
    )r   r   r   r   r   r   )r'   r   r	   r   r   )rd   re   r   r   r   Ústudent_unwrappedr   r   s           r>   Ú_resolve_teacherrh   ¤   se   € õ& �'Õ.Ñ/Ô/ð Øˆõ % ]Ñ3Ô3ÐØ#Ô/€KØ Ô)€HåØ$ØØØ'ØØðñ ô ð r?   c                   ó‚  ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 ddej        deeej        ef         de	ej                 d	e	e         d
ede	e
         de	e
         de
de	ej                 de	ej                 defˆ fd„Z	 dde	e         dd fd„Z	 	 ddede	e         dej        fd„Zdej        dej        deeej        f         fd„Zˆ xZS )ÚLogitDistillationTaska  Logit-based knowledge distillation task.

    Performs distillation by matching student and teacher output logits using
    KL divergence with temperature scaling.

    Loss weighting supports two modes:
    1. Independent weights: loss = task_loss_weight * task_loss + distill_loss_weight * distill_loss
    2. Complementary mode: loss = task_loss_weight * task_loss + (1 - task_loss_weight) * distill_loss
       (used when only task_loss_weight is specified)

    Args:
        student_model: Student model to train
        teacher_model: Teacher model - can be a model name string, nn.Module, or DistillationTeacher
        criterion: Task loss function (default: CrossEntropyLoss)
        teacher_pretrained_path: Path to teacher pretrained weights (used when teacher_model is a string)
        loss_type: Type of distillation loss (currently only 'kl' supported)
        distill_loss_weight: Weight for distillation loss
        task_loss_weight: Weight for task loss
        temperature: Softmax temperature for distillation (typical values: 1-4)
        device: Device for task tensors/buffers
        dtype: Dtype for task tensors/buffers
        verbose: Enable info logging

    Example:
        >>> # With model name string (num_classes/in_chans inferred from student)
        >>> task = LogitDistillationTask(
        ...     student_model=model, teacher_model='resnet50',
        ...     criterion=nn.CrossEntropyLoss(),
        ...     task_loss_weight=0.3, temperature=4.0,
        ...     device=torch.device('cuda'),
        ... )
        >>> # With raw model
        >>> task = LogitDistillationTask(
        ...     student_model=model, teacher_model=my_teacher_model,
        ...     criterion=nn.CrossEntropyLoss(),
        ...     task_loss_weight=0.3, temperature=4.0,
        ... )
    NÚklç      ð?Tre   Úteacher_modelÚ	criterionÚteacher_pretrained_pathÚ	loss_typeÚdistill_loss_weightÚtask_loss_weightÚtemperaturer   r   Úverbosec                 ó„  •— t          ¦   «                              |	|
|¬¦  «         t          |||| j        | j        ¦  «        }|| _        || _        |�|nt          j        ¦   «         | _	        || _
        || _        |dk    rt          d|› d�¦  «        ‚t          |¦  «        }t          j        |j        d         | j        | j        ¬¦  «                             dddd¦  «        }t          j        |j        d	         | j        | j        ¬¦  «                             dddd¦  «        }|                      d
|d¬¦  «         |                      d|d¬¦  «         |�?|| _        |�|nd| _        | j        r%t,                               d| j        › d|› �¦  «         n|�>|| _        d|z
  | _        | j        r%t,                               d|› d| j        › �¦  «         n?d| _        d| _        | j        r*t,                               d| j        › d| j        › �¦  «         | j        r"t,                               d|› d|› �¦  «         d S d S )N©r   r   rt   rk   zUnsupported loss_type 'z$'. Currently only 'kl' is supported.r   r   r
   r   r   rV   Fr!   rW   rl   z9LogitDistillationTask: Independent weights - task_weight=ú, distill_weight=z8LogitDistillationTask: Complementary mode - task_weight=z;LogitDistillationTask: Default equal weights - task_weight=z!LogitDistillationTask: loss_type=z, temperature=)r%   r&   rh   r   r   Útrainable_modulerd   r,   ÚCrossEntropyLossrn   rp   rs   rG   r	   r5   r6   r   r7   r8   rq   rr   rt   r)   r*   )r:   re   rm   rn   ro   rp   rq   rr   rs   r   r   rt   rd   rg   rV   rW   r=   s                   €r>   r&   zLogitDistillationTask.__init__ñ   sí  ø€ õ 	‰Œ×Ò ¨e¸WÐÑEÔEÐEõ #ØØØ#ØŒKØŒJñ
ô 
ˆð !.ˆÔØˆŒØ&/Ð&;˜˜ÅÔATÑAVÔAVˆŒØ"ˆŒØ&ˆÔà˜ÒÐÝÐf°yÐfÐfÐfÑgÔgÐgõ )¨Ñ7Ô7ÐÝ”|ØÔ,¨VÔ4Ø”;Ø”*ð
ñ 
ô 
÷ Š$ˆq�"�a˜Ñ
Ô
ð	 	õ
 ”lØÔ,¨UÔ3Ø”;Ø”*ð
ñ 
ô 
÷ Š$ˆq�"�a˜Ñ
Ô
ð	 	ð
 	×Ò˜^¨\ÀeÐÑLÔLÐLØ×Ò˜]¨KÀEÐÑJÔJÐJð Ð*à':ˆDÔ$Ø8HÐ8TÐ$4Ð$4ÐZ]ˆDÔ!ØŒ|ð Ý—’ðaØ#'Ô#8ðað aØK^ðað añô ð øð Ð)à$4ˆDÔ!Ø'*Ð-=Ñ'=ˆDÔ$ØŒ|ð Ý—’ðaØ#3ðað aØFJÔF^ðað añô ð øð (+ˆDÔ$Ø$'ˆDÔ!ØŒ|ð Ý—’ðfØ#'Ô#8ðfð fØKOÔKcðfð fñô ð ð
 Œ<ð 	Ý�LŠLØZ°IÐZÐZÈ[ÐZÐZñô ð ð ð ð	ð 	r?   Ú
device_idsrA   c                 ó‚   — ddl m} | j                             ¦   «         D ]	}d|_        Œ
 || j        fd|i|¤Ž| _        | S )aŒ  Prepare task for distributed training.

        Wraps the student model in DistributedDataParallel (DDP) while leaving
        the frozen teacher model unwrapped.

        Args:
            device_ids: List of device IDs for DDP (e.g., [local_rank])
            **ddp_kwargs: Additional arguments passed to DistributedDataParallel

        Returns:
            self (for method chaining)
        r   ©ÚDistributedDataParallelFrz   ©Útorch.nn.parallelr}   rd   Ú
parametersÚrequires_gradrx   ©r:   rz   Ú
ddp_kwargsÚDDPÚparams        r>   Úprepare_distributedz)LogitDistillationTask.prepare_distributedD  óg   € ð" 	EÐDÐDÐDÐDÐDà”\×,Ò,Ñ.Ô.ð 	(ð 	(ˆEØ"'ˆEÔÐà #  DÔ$9Ð _Ð _ÀjÐ _ÐT^Ð _Ð _ˆÔØˆr?   rJ   rK   rL   c                 ó”   — t          j        | j        f||dœ|¤Ž| _        | j        | _         | j        j        d||dddœ|¤Ž | j        S )z=Compile student eval/train forward and teacher logit forward.rP   TF©rK   rL   rM   rN   r$   )r5   rQ   rx   Ú
eval_modelrd   )r:   rK   rL   rR   s       r>   rQ   zLogitDistillationTask.compile]  sx   € õ !&¤¨dÔ.CÐ rÈWÐ[_Ð rÐ rÐcqÐ rÐ rˆÔàÔ/ˆŒØˆŒÔð 	
ØØØØ"ð		
ð 	
ð
 ð	
ð 	
ð 	
ð Ô$Ð$r?   r@   Útargetc                 óB  — |                       |¦  «        }|                      ||¦  «        }t          j        ¦   «         5  | j                             || j        | j        ¦  «        }|                      |                     ¦   «         d¬¦  «        }ddd¦  «         n# 1 swxY w Y   t          j
        || j        z  d¬¦  «        }t          j
        || j        z  d¬¦  «        }t          j        ||dd¬¦  «        | j        d	z  z  }	| j        |z  | j        |	z  z   }
|
|||	d
œS )aÁ  Forward pass with logit distillation.

        Args:
            input: Input tensor [B, C, H, W]
            target: Target labels [B]

        Returns:
            Dictionary containing:
                - 'loss': Combined training loss (task + distillation)
                - 'output': Student logits (for metrics)
                - 'task_loss': Classification loss component
                - 'kd_loss': Logit distillation loss component
        F©rS   Nr   )ÚdimÚ	batchmeanT)Ú	reductionÚ
log_targeté   ©ÚlossÚoutputÚ	task_lossÚkd_loss)rx   rn   r5   Úno_gradrd   rZ   rV   rW   ÚdetachÚFÚlog_softmaxrs   Úkl_divrr   rq   )r:   r@   r‹   Ústudent_logitsr–   Úinput_kdÚteacher_logitsÚprob_sÚprob_tr—   Ú
total_losss              r>   rU   zLogitDistillationTask.forwardp  ss  € ð$ ×.Ò.¨uÑ5Ô5ˆØ—N’N >°6Ñ:Ô:ˆ	åŒ]‰_Œ_ð 	Tð 	TØ”|×3Ò3°E¸4Ô;LÈdÔN^Ñ_Ô_ˆHØ!Ÿ\š\¨(¯/ª/Ñ*;Ô*;ÈU˜\ÑSÔSˆNð	Tð 	Tð 	Tñ 	Tô 	Tð 	Tð 	Tð 	Tð 	Tð 	Tð 	Tøøøð 	Tð 	Tð 	Tð 	Tõ ”˜~°Ô0@Ñ@ÀbÐIÑIÔIˆÝ”˜~°Ô0@Ñ@ÀbÐIÑIÔIˆÝ”(˜6 6°[ÈTÐRÑRÔRÐVZÔVfÐjkÑVkÑlˆàÔ*¨YÑ6¸Ô9QÐT[Ñ9[Ñ[ˆ
ð Ø$Ø"Øð	
ð 
ð 	
s   ¿ABÂBÂ"B)	NNrk   NNrl   NNT©N©rJ   N)r0   r\   r]   r^   r,   r-   r   r(   r   r   Úfloatr5   r   r   ra   r&   Úlistr†   rQ   r`   r   rU   rb   rc   s   @r>   rj   rj   É   sÒ  ø€ € € € € ð%ð %ðV .2Ø59Ø!Ø37Ø04Ø!$Ø-1Ø+/Ø ðQð Qàœ9ðQð !  b¤iÐ1DÐ!DÔEðQð   ¤	Ô*ð	Qð
 &.¨c¤]ðQð ðQð "*¨%¤ðQð ' uœoðQð ðQð ˜Uœ\Ô*ðQð ˜EœKÔ(ðQð ðQð Qð Qð Qð Qð Qðj *.ðð à  œðð 
!ð	ð ð ð ð6 &Ø"&ð%ð %àð%ð ˜3”-ð%ð
 
Œð%ð %ð %ð %ð&$
à”<ð$
ð ”Lð$
ð 
ˆc�5”<ÐÔ	 ð	$
ð $
ð $
ð $
ð $
ð $
ð $
ð $
r?   rj   c                   ó¢   ‡ — e Zd ZdZ	 ddej        deej                 fˆ fd„Zd„ Zdde	fd	„Z
d
ej        deej        ej        f         fd„Zˆ xZS )Ú"FeatureDistillationTrainableModulea  Trainable module for feature distillation.

    Wraps student model and projection layer into a single module where all
    trainable forward operations happen inside forward(). This ensures proper
    DDP wrapping when the module is used with DistributedDataParallel.
    Nre   Ú
projectionc                 ód   •— t          ¦   «                              ¦   «          || _        || _        dS )zÍ Create trainable module wrapper for feature distillation.

        Args:
            student_model: Student model to train
            projection: Optional projection layer (Linear layer or None)
        N)r%   r&   Ústudentr©   )r:   re   r©   r=   s      €r>   r&   z+FeatureDistillationTrainableModule.__init__Ÿ  s-   ø€ õ 	‰Œ×ÒÑÔÐØ$ˆŒØ$ˆŒˆˆr?   c                 ó¢   — t          | j        ¦  «        }t          |d¦  «        st          ¦   «         S d„ |                     ¦   «         D ¦   «         S )NÚno_weight_decayc                 ó   — h | ]}d |z   ’ŒS )ústudent.r$   )Ú.0Únames     r>   ú	<setcomp>zEFeatureDistillationTrainableModule.no_weight_decay.<locals>.<setcomp>²  s   € ÐHÐHÐH d�
˜TÑ!ÐHÐHÐHr?   )r	   r«   r3   Úsetr­   )r:   r«   s     r>   r­   z2FeatureDistillationTrainableModule.no_weight_decay®  sM   € Ý˜tœ|Ñ,Ô,ˆÝ�wÐ 1Ñ2Ô2ð 	Ý‘5”5ˆLØHÐH¨g×.EÒ.EÑ.GÔ.GÐHÑHÔHÐHr?   FÚcoarsec                 óô   ‡‡— t          | j        ¦  «        }t          |d¦  «        si S t          ||                     |¬¦  «        d¬¦  «        Št          ‰                     ¦   «         d¬¦  «        Šˆˆfd„}|S )NÚgroup_matcher)r´   T)Úreverser   )Údefaultc                 óˆ   •— |                       d¦  «        r+‰                     | t          d¦  «        d …         ‰¦  «        S ‰S )Nr¯   )Ú
startswithr4   Úlen)r±   Ústudent_layer_mapÚ
task_layers    €€r>   Ú_matcherzBFeatureDistillationTrainableModule.group_matcher.<locals>._matcher¼  sE   ø€ Ø�Š˜zÑ*Ô*ð QØ(×,Ò,¨Tµ#°j±/´/Ð2BÐ2BÔ-CÀZÑPÔPÐPØÐr?   )r	   r«   r3   r   r¶   ÚmaxÚvalues)r:   r´   r«   r¾   r¼   r½   s       @@r>   r¶   z0FeatureDistillationTrainableModule.group_matcher´  s–   øø€ Ý˜tœ|Ñ,Ô,ˆÝ�w Ñ0Ô0ð 	ØˆIå,¨W°g×6KÒ6KÐSYÐ6KÑ6ZÔ6ZÐdhÐiÑiÔiÐÝÐ*×1Ò1Ñ3Ô3¸QÐ?Ñ?Ô?ˆ
ð	ð 	ð 	ð 	ð 	ð 	ð
 ˆr?   r@   rA   c                 óâ   — | j                              |¦  «        }| j                              |¦  «        }| j                              |d¬¦  «        }| j        �|                      |¦  «        }||fS )a  Forward pass through student and projection.

        Args:
            input: Input tensor [B, C, H, W]

        Returns:
            Tuple of (student_logits, student_features) where features are
            optionally projected to match teacher dimension.
        TrE   )r«   rC   rD   r©   )r:   r@   rH   r�   Ústudent_featuress        r>   rU   z*FeatureDistillationTrainableModule.forwardÃ  sp   € ð ”l×3Ò3°EÑ:Ô:ˆØœ×2Ò2°;Ñ?Ô?ˆØœ<×4Ò4°[ÈTÐ4ÑRÔRÐàŒ?Ð&Ø#ŸšÐ/?Ñ@Ô@ÐàÐ/Ð/Ð/r?   r£   r[   )r0   r\   r]   r^   r,   r-   r   r&   r­   ra   r¶   r5   r`   r   rU   rb   rc   s   @r>   r¨   r¨   —  sÐ   ø€ € € € € ðð ð /3ð%ð %àœ9ð%ð ! ¤Ô+ð%ð %ð %ð %ð %ð %ðIð Ið Iðð  Dð ð ð ð ð0˜Uœ\ð 0¨e°E´LÀ%Ä,Ð4NÔ.Oð 0ð 0ð 0ð 0ð 0ð 0ð 0ð 0r?   r¨   c                   óâ  ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 d&dej        deeej        ef         de	ej                 de	e         de	e
         d	e	e
         d
e	e         de	e         de	ej                 de	ej                 defˆ fd„Zedej        defd„¦   «         Z	 d'de	e         dd fd„Z	 	 d(dede	e         dej        fd„Zd)de	ej                 dede	ej                 fˆ fd„Z	 	 d)de	ej                 dedeeeeej        f         f         fd„Zd*defd„Z	 	 	 d+d e	eeeeej        f         f                  d!ede	ej                 deddf
d"„Zd#ej        d$ej        deeej        f         fd%„Zˆ xZS ),ÚFeatureDistillationTaska�  Feature-based knowledge distillation task.

    Performs distillation by matching student and teacher intermediate features
    (pooled pre-logits) using MSE loss. Automatically creates a projection layer
    if student and teacher feature dimensions differ.

    Loss weighting supports two modes:
    1. Independent weights: loss = task_loss_weight * task_loss + distill_loss_weight * distill_loss
    2. Complementary mode: loss = task_loss_weight * task_loss + (1 - task_loss_weight) * distill_loss
       (used when only task_loss_weight is specified)

    Args:
        student_model: Student model to train
        teacher_model: Teacher model - can be a model name string, nn.Module, or DistillationTeacher
        criterion: Task loss function (default: CrossEntropyLoss)
        teacher_pretrained_path: Path to teacher pretrained weights (used when teacher_model is a string)
        distill_loss_weight: Weight for distillation loss
        task_loss_weight: Weight for task loss
        student_feature_dim: Student pre-logits dimension (auto-detected if None)
        teacher_feature_dim: Teacher pre-logits dimension (auto-detected if None)
        device: Device for task tensors/buffers
        dtype: Dtype for task tensors/buffers
        verbose: Enable info logging

    Example:
        >>> # With model name string (num_classes/in_chans inferred from student)
        >>> task = FeatureDistillationTask(
        ...     student_model=model, teacher_model='resnet50',
        ...     criterion=nn.CrossEntropyLoss(),
        ...     distill_loss_weight=5.0, task_loss_weight=1.0,
        ...     device=torch.device('cuda'),
        ... )
    NTre   rm   rn   ro   rq   rr   Ústudent_feature_dimÚteacher_feature_dimr   r   rt   c                 óž  •— t          ¦   «                              |	|
|¬¦  «         t          |||| j        | j        ¦  «        }|| _        |�|nt          j        ¦   «         | _        |�?|| _	        |�|nd| _
        | j        r%t                               d| j
        › d|› �¦  «         n|�>|| _
        d|z
  | _	        | j        r%t                               d|› d| j	        › �¦  «         n?d| _	        d| _
        | j        r*t                               d| j
        › d| j	        › �¦  «         |€|                      |¦  «        }|€|                      |j        ¦  «        }d }||k    rJ| j        r t                               d|› d|› �¦  «         t          j        ||| j        | j        ¬	¦  «        }n!| j        rt                               d
¦  «         t#          ||¦  «        | _        t'          |¦  «        }t)          j        |j        d         | j        | j        ¬	¦  «                             dddd¦  «        }t)          j        |j        d         | j        | j        ¬	¦  «                             dddd¦  «        }|                      d|d¬¦  «         |                      d|d¬¦  «         | j        r"t                               d|› d|› �¦  «         d S d S )Nrv   rl   z;FeatureDistillationTask: Independent weights - task_weight=rw   z:FeatureDistillationTask: Complementary mode - task_weight=z=FeatureDistillationTask: Default equal weights - task_weight=zCreating projection layer: z -> r   z.Feature dimensions match, no projection neededr   r
   r   r   rV   Fr!   rW   z%FeatureDistillationTask: student_dim=z, teacher_dim=)r%   r&   rh   r   r   rd   r,   ry   rn   rq   rr   rt   r)   r*   Ú_detect_feature_dimr2   ÚLinearr¨   rx   r	   r5   r6   r   r7   r8   )r:   re   rm   rn   ro   rq   rr   rÅ   rÆ   r   r   rt   rd   r©   rg   rV   rW   r=   s                    €r>   r&   z FeatureDistillationTask.__init__ú  s—  ø€ õ 	‰Œ×Ò ¨e¸WÐÑEÔEÐEõ #ØØØ#ØŒKØŒJñ
ô 
ˆð ˆŒØ&/Ð&;˜˜ÅÔATÑAVÔAVˆŒð Ð*à':ˆDÔ$Ø8HÐ8TÐ$4Ð$4ÐZ]ˆDÔ!ØŒ|ð Ý—’ðaØ#'Ô#8ðað aØK^ðað añô ð øð Ð)à$4ˆDÔ!Ø'*Ð-=Ñ'=ˆDÔ$ØŒ|ð Ý—’ðaØ#3ðað aØFJÔF^ðað añô ð øð (+ˆDÔ$Ø$'ˆDÔ!ØŒ|ð Ý—’ðfØ#'Ô#8ðfð fØKOÔKcðfð fñô ð ð Ð&Ø"&×":Ò":¸=Ñ"IÔ"IÐØÐ&Ø"&×":Ò":¸7¼=Ñ"IÔ"IÐð ˆ
ØÐ"5Ò5Ð5ØŒ|ð Ý—’Ø`Ð2EÐ`Ð`ÐK^Ð`Ð`ñô ð õ œÐ#6Ð8KÐTXÔT_ÐgkÔgqÐrÑrÔrˆJˆJàŒ|ð OÝ—’ÐMÑNÔNÐNå BÀ=ÐR\Ñ ]Ô ]ˆÔõ )¨Ñ7Ô7ÐÝ”|ØÔ,¨VÔ4Ø”;Ø”*ð
ñ 
ô 
÷ Š$ˆq�"�a˜Ñ
Ô
ð	 	õ
 ”lØÔ,¨UÔ3Ø”;Ø”*ð
ñ 
ô 
÷ Š$ˆq�"�a˜Ñ
Ô
ð	 	ð
 	×Ò˜^¨\ÀeÐÑLÔLÐLØ×Ò˜]¨KÀEÐÑJÔJÐJàŒ<ð 	Ý�LŠLðXØ2ðXð XØBUðXð Xñô ð ð ð ð	ð 	r?   r2   rA   c                 óš   — t          | ¦  «        } t          | d¦  «        r| j        S t          | d¦  «        r| j        S t	          d¦  «        ‚)z)Auto-detect feature dimension from model.Úhead_hidden_sizeÚnum_featuresz±Cannot auto-detect feature dimension. Model must have 'head_hidden_size' or 'num_features' attribute, or you must specify student_feature_dim and teacher_feature_dim explicitly.)r	   r3   rË   rÌ   rG   )r2   s    r>   rÈ   z+FeatureDistillationTask._detect_feature_dim\  s`   € õ ˜UÑ#Ô#ˆå�5Ð,Ñ-Ô-ð 		ØÔ)Ð)Ý�U˜NÑ+Ô+ð 	ØÔ%Ð%åðRñô ð r?   rz   c                 ó‚   — ddl m} | j                             ¦   «         D ]	}d|_        Œ
 || j        fd|i|¤Ž| _        | S )a¦  Prepare task for distributed training.

        Wraps the trainable module (student + projection) in DistributedDataParallel
        (DDP) while leaving the frozen teacher model unwrapped.

        Args:
            device_ids: List of device IDs for DDP (e.g., [local_rank])
            **ddp_kwargs: Additional arguments passed to DistributedDataParallel

        Returns:
            self (for method chaining)
        r   r|   Frz   r~   r‚   s        r>   r†   z+FeatureDistillationTask.prepare_distributedl  r‡   r?   rJ   rK   rL   c                 ó¾   — t          j        | j        j        f||dœ|¤Ž}|| _        t          j        | j        f||dœ|¤Ž| _         | j        j        d||dddœ|¤Ž |S )z9Compile feature-distillation train and eval entry points.rP   FTr‰   r$   )r5   rQ   rx   r«   rŠ   rd   )r:   rK   rL   rR   rŠ   s        r>   rQ   zFeatureDistillationTask.compile…  s¨   € õ ”] 4Ô#8Ô#@ÐoÈ'ÐX\ÐoÐoÐ`nÐoÐoˆ
Ø$ˆŒÝ %¤ØÔ!ð!
àØð!
ð !
ð ð	!
ð !
ˆÔð 	ˆŒÔð 	
ØØØØ!ð		
ð 	
ð
 ð	
ð 	
ð 	
ð Ðr?   FÚmoduleÚemac                 óÂ   •— t          ¦   «                              ||¬¦  «        }|}|�6t          |t          ¦  «        r|j        S t          |d¦  «        sn	|j        }|­6|S )N)rÏ   rÐ   rÏ   )r%   Úget_eval_modelr'   r¨   r«   r3   rÏ   )r:   rÏ   rÐ   ÚresolvedÚcurrentr=   s        €r>   rÒ   z&FeatureDistillationTask.get_eval_model�  sp   ø€ Ý‘7”7×)Ò)°¸SÐ)ÑAÔAˆØˆØÐ!Ý˜'Õ#EÑFÔFð 'Ø”Ð&Ý˜7 HÑ-Ô-ð ØØ”nˆGð Ð!ð ˆr?   c                 ó¤   — |�|n|                       |¬¦  «        }|€i S t          |¦  «        }|j        €i S d|j                             ¦   «         iS )N©rÐ   r©   )Úget_trainable_moduler	   r©   Ú
state_dict)r:   rÏ   rÐ   Ú	trainables       r>   Úget_task_statez&FeatureDistillationTask.get_task_state¨  sc   € ð
 "Ð-��°4×3LÒ3LÐQTÐ3LÑ3UÔ3UˆØˆ>ØˆIÝ  Ñ(Ô(ˆ	ØÔÐ'ØˆIØ˜iÔ2×=Ò=Ñ?Ô?Ð@Ð@r?   Úexclude_headc                 óø   — t          | j        ¦  «        }t          |j                             ¦   «         ¦  «        }|r
|d d…         }|j        �,|                     |j                             ¦   «         ¦  «         |S )Néþÿÿÿ)r	   rx   r¦   r«   r€   r©   Úextend)r:   rÛ   rÙ   r€   s       r>   Úget_clip_parametersz+FeatureDistillationTask.get_clip_parametersµ  su   € Ý  Ô!6Ñ7Ô7ˆ	Ý˜)Ô+×6Ò6Ñ8Ô8Ñ9Ô9ˆ
Øð 	)Ø# C R CœˆJØÔÐ+Ø×Ò˜iÔ2×=Ò=Ñ?Ô?Ñ@Ô@Ð@ØÐr?   ÚstateÚstrictc                 óB  — |sd S |�|n|                       |¬¦  «        }|€ |rt          d¦  «        ‚t          d¦  «        ‚t          |¦  «        }|                     d¦  «        }|€d S |j        €|rt          d¦  «        ‚d S |j                             ||¬¦  «         d S )NrÖ   z.Cannot load EMA task state before setup_ema().z2Cannot load task state without a trainable module.r©   z@Checkpoint has projection task state but task has no projection.)rá   )r×   ÚRuntimeErrorr	   r4   r©   Úload_state_dict)r:   rà   rá   rÏ   rÐ   rÙ   Úprojection_states          r>   Úload_task_statez'FeatureDistillationTask.load_task_state¾  sÐ   € ð ð 	ØˆFØ!Ð-��°4×3LÒ3LÐQTÐ3LÑ3UÔ3UˆØˆ>Øð UÝ"Ð#SÑTÔTÐTÝÐSÑTÔTÐTÝ  Ñ(Ô(ˆ	Ø Ÿ9š9 \Ñ2Ô2ÐØÐ#ØˆFØÔÐ'Øð gÝ"Ð#eÑfÔfÐfØˆFØÔ×,Ò,Ð-=ÀfÐ,ÑMÔMÐMÐMÐMr?   r@   r‹   c                 ó´  — |                       |¦  «        \  }}|                      ||¦  «        }t          j        ¦   «         5  | j                             || j        | j        ¦  «        }|                      |                     ¦   «         d¬¦  «        }ddd¦  «         n# 1 swxY w Y   t          j
        ||¦  «        }| j        |z  | j        |z  z   }	|	|||dœS )aÅ  Forward pass with feature distillation.

        Args:
            input: Input tensor [B, C, H, W]
            target: Target labels [B]

        Returns:
            Dictionary containing:
                - 'loss': Combined training loss (task + distillation)
                - 'output': Student logits (for metrics)
                - 'task_loss': Classification loss component
                - 'kd_loss': Feature distillation loss component
        Tr�   Nr“   )rx   rn   r5   r˜   rd   rZ   rV   rW   r™   rš   Úmse_lossrr   rq   )
r:   r@   r‹   r�   rÂ   r–   rž   Úteacher_featuresr—   r¢   s
             r>   rU   zFeatureDistillationTask.forwardÖ  s.  € ð$ ,0×+@Ò+@ÀÑ+GÔ+GÑ(ˆÐ(Ø—N’N >°6Ñ:Ô:ˆ	åŒ]‰_Œ_ð 	Uð 	UØ”|×3Ò3°E¸4Ô;LÈdÔN^Ñ_Ô_ˆHØ#Ÿ|š|¨H¯OªOÑ,=Ô,=Èt˜|ÑTÔTÐð	Uð 	Uð 	Uñ 	Uô 	Uð 	Uð 	Uð 	Uð 	Uð 	Uð 	Uøøøð 	Uð 	Uð 	Uð 	Uõ ”*Ð-Ð/?Ñ@Ô@ˆØÔ*¨YÑ6¸Ô9QÐT[Ñ9[Ñ[ˆ
ð Ø$Ø"Øð	
ð 
ð 	
s   ÁABÂB"Â%B")	NNNNNNNNTr£   r¤   )NFr[   )TNF)r0   r\   r]   r^   r,   r-   r   r(   r   r   r¥   r_   r5   r   r   ra   r&   ÚstaticmethodrÈ   r¦   r†   rQ   rÒ   r   r`   rÚ   rß   ræ   rU   rb   rc   s   @r>   rÄ   rÄ   ×  sU  ø€ € € € € ð ð  ðL .2Ø59Ø37Ø04Ø15Ø15Ø-1Ø+/Ø ð`ð `àœ9ð`ð !  b¤iÐ1DÐ!DÔEð`ð   ¤	Ô*ð	`ð
 &.¨c¤]ð`ð "*¨%¤ð`ð ' uœoð`ð "*¨#¤ð`ð "*¨#¤ð`ð ˜Uœ\Ô*ð`ð ˜EœKÔ(ð`ð ð`ð `ð `ð `ð `ð `ðD ð 2¤9ð °ð ð ð ñ „\ðð" *.ðð à  œðð 
#ð	ð ð ð ð6 &Ø"&ðð àðð ˜3”-ðð
 
Œðð ð ð ð0	ð 	 X¨b¬iÔ%8ð 	Àdð 	ÐW_Ð`bÔ`iÔWjð 	ð 	ð 	ð 	ð 	ð 	ð +/ØðAð Aà˜RœYÔ'ðAð ðAð 
ˆc�4˜˜Uœ\Ð)Ô*Ð*Ô	+ð	Að Að Að Aðð °ð ð ð ð ð  Ø*.ØðNð Nà˜D  d¨3°´Ð+<Ô&=Ð!=Ô>Ô?ðNð ðNð ˜RœYÔ'ð	Nð
 ðNð 
ðNð Nð Nð Nð0!
à”<ð!
ð ”Lð!
ð 
ˆc�5”<ÐÔ	 ð	!
ð !
ð !
ð !
ð !
ð !
ð !
ð !
r?   rÄ   ) r^   ÚloggingÚtypingr   r   r   r   r5   Útorch.nnr,   Útorch.nn.functionalÚ
functionalrš   Útimm.modelsr   r   Ú
timm.utilsr	   Útaskr   Ú	getLoggerr0   r)   r-   r   r(   r   r   rh   rj   r¨   rÄ   r$   r?   r>   ú<module>rô      s	  ðØ ;Ð ;Ø €€€Ø /Ð /Ð /Ð /Ð /Ð /Ð /Ð /Ð /Ð /Ð /Ð /à €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð à 6Ð 6Ð 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø #Ð #Ð #Ð #Ð #Ð #à Ð Ð Ð Ð Ð ð ˆ'Ô
˜HÑ
%Ô
%€ðOQð OQð OQð OQð OQ˜"œ)ñ OQô OQð OQðd"Ø�s˜BœIÐ':Ð:Ô;ð"à”yð"ð " #œð"ð ˜œÔ&ð	"ð
 ˜œÔ$ð"ð ð"ð "ð "ð "ðJK
ð K
ð K
ð K
ð K
˜Lñ K
ô K
ð K
ð\=0ð =0ð =0ð =0ð =0¨¬ñ =0ô =0ð =0ð@`
ð `
ð `
ð `
ð `
˜lñ `
ô `
ð `
ð `
ð `
r?   