§
    ·¯¸j~;  ã                   óÒ   — d Z ddlZddlmZmZmZ ddlZddlmZ ddl	mc m
Z ddlmZ ddlmZ ddlmZ  ej        e¦  «        Z G d„ d	ej        ¦  «        Z G d
„ de¦  «        ZdS )zJToken-based distillation training task for models with distillation heads.é    N)ÚDictÚOptionalÚUnion)Úcreate_model)Úunwrap_modelé   )ÚTrainingTaskc                   óN  ‡ — e Zd ZdZ	 	 	 	 	 ddeeej        f         dee	         de	dee         dee
j                 d	ee
j                 fˆ fd
„Z	 	 ddedee         dd fd„Zde
j        de
j        fd„Z	 	 dde
j        dee
j                 dee
j                 de
j        fd„Zˆ xZS )ÚTokenDistillationTeacheraK  Wrapper for a teacher model used in token-based distillation.

    Creates and manages a pre-trained teacher model for token distillation,
    handling model creation and normalization differences between teacher and student.

    Can be created from:
    - A model name string (creates the model internally)
    - An existing nn.Module (wraps it with the necessary interface)

    Args:
        model_name_or_module: Either a model name string or an nn.Module
        num_classes: Number of output classes (required if model_name_or_module is a string)
        in_chans: Number of input channels (used if model_name_or_module is a string)
        pretrained_path: Optional path to pretrained weights (used if model_name_or_module is a string)
        device: Device to place the model on
        dtype: Model dtype (uses float32 if None)
    Né   Úmodel_name_or_moduleÚnum_classesÚin_chansÚpretrained_pathÚdeviceÚdtypec           	      óˆ  •— t          ¦   «                              ¦   «          t          |t          ¦  «        rKt                               d|› d�¦  «         ddi}|rt          ||¬¦  «        |d<   t          d|||||dœ|¤Ž}nAt          |t          j	        ¦  «        r|}n$t          dt          |¦  «        j        › �¦  «        ‚|                     ¦   «          || _        t          |¦  «        }	t!          |	d	¦  «        r7|	j                             d
d¦  «        }
|	j                             dd¦  «        }nd}
d}t'          j        |
||¬¦  «                             dddd¦  «        }t'          j        |||¬¦  «                             dddd¦  «        }|                      d|d¬¦  «         |                      d|d¬¦  «         d S )Nz,Creating token distillation teacher model: 'ú'Ú
pretrainedT)Úfiler   Úpretrained_cfg_overlay)Ú
model_namer   r   r   r   z8model_name_or_module must be a string or nn.Module, got Úpretrained_cfgÚmean)g
×£p=
ß?gÉv¾Ÿ/Ý?g–C‹lçûÙ?Ústd)gZd;ßOÍ?gyé&1¬Ì?gÍÌÌÌÌÌÌ?©r   r   r   éÿÿÿÿÚmean_kdF©Ú
persistentÚstd_kd© )ÚsuperÚ__init__Ú
isinstanceÚstrÚ_loggerÚinfoÚdictr   ÚnnÚModuleÚ	TypeErrorÚtypeÚ__name__ÚevalÚmodelr   Úhasattrr   ÚgetÚtorchÚtensorÚviewÚregister_buffer)Úselfr   r   r   r   r   r   Úpretrained_kwargsr0   Úmodel_unwrappedr   r   r   r!   Ú	__class__s                 €úl/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/timm/task/token_distillation.pyr$   z!TokenDistillationTeacher.__init__$   s  ø€ õ 	‰Œ×ÒÑÔÐåÐ*­CÑ0Ô0ð 	Ý�LŠLÐ_ÐH\Ð_Ð_Ð_Ñ`Ô`Ð`à!-¨tÐ 4ÐØð Ý>BØ(Ø +ð?ñ ?ô ?Ð!Ð":Ñ;õ
 !ð Ø/Ø'Ø!ØØðð ð $ðð ˆEˆEõ Ð,­b¬iÑ8Ô8ð 	Ø(ˆEˆEåØpÍ4ÐPdÑKeÔKeÔKnÐpÐpñô ð ð 	�
Š
‰ŒˆØˆŒ
õ ' uÑ-Ô-ˆÝ�?Ð$4Ñ5Ô5ð 	(Ø"Ô1×5Ò5°fÐ>SÑTÔTˆDØ!Ô0×4Ò4°UÐ<QÑRÔRˆCˆCà(ˆDØ'ˆCå”,˜t¨F¸%Ð@Ñ@Ô@×EÒEÀaÈÈQÐPQÑRÔRˆÝ”˜c¨&¸Ð>Ñ>Ô>×CÒCÀAÀrÈ1ÈaÑPÔPˆØ×Ò˜Y¨¸EÐÑBÔBÐBØ×Ò˜X v¸%ÐÑ@Ô@Ð@Ð@Ð@ó    ÚinductorÚbackendÚmodeÚreturnc                 óD   — t          j        | j        f||dœ|¤Ž| _        | S )z Compile teacher logit inference.©r>   r?   )r3   Úcompiler0   ©r7   r>   r?   Úcompile_kwargss       r;   rC   z TokenDistillationTeacher.compileY   s+   € õ ”] 4¤:Ð\°wÀTÐ\Ð\È^Ð\Ð\ˆŒ
Øˆr<   Úinputc                 ó,   — |                       |¦  «        S )z³Forward pass through teacher model.

        Args:
            input: Input tensor (should already be normalized for teacher)

        Returns:
            Teacher logits
        )r0   )r7   rF   s     r;   Úforwardz TokenDistillationTeacher.forwardc   s   € ð �zŠz˜%Ñ Ô Ð r<   Ústudent_meanÚstudent_stdc                 óª   — |�|€|S t          j        || j        ¦  «        rt          j        || j        ¦  «        r|S ||z  |z   | j        z
  | j        z  S )ao  Normalize input to match teacher's expected normalization.

        Args:
            input: Input tensor (already normalized for student)
            student_mean: Student normalization mean buffer [1, 3, 1, 1]
            student_std: Student normalization std buffer [1, 3, 1, 1]

        Returns:
            Input tensor normalized for the teacher model
        )r3   Úequalr   r!   )r7   rF   rI   rJ   s       r;   Únormalize_inputz(TokenDistillationTeacher.normalize_inputn   sd   € ð  Ð ;Ð#6ØˆLÝŒ;�| T¤\Ñ2Ô2ð 	µu´{À;ÐPTÔP[Ñ7\Ô7\ð 	ØˆLØ˜Ñ# lÑ2°T´\ÑAÀTÄ[ÑPÐPr<   )Nr   NNN©r=   N)NN)r.   Ú
__module__Ú__qualname__Ú__doc__r   r&   r*   r+   r   Úintr3   r   r   r$   rC   ÚTensorrH   rM   Ú__classcell__©r:   s   @r;   r   r      s“  ø€ € € € € ðð ð* *.ØØ-1Ø-1Ø+/ð3Að 3Aà"'¨¨R¬Y¨Ô"7ð3Að " #œð3Að ð	3Að
 & cœ]ð3Að ˜Uœ\Ô*ð3Að ˜EœKÔ(ð3Að 3Að 3Að 3Að 3Að 3Aðn &Ø"&ðð àðð ˜3”-ðð
 
$ðð ð ð ð	!˜Uœ\ð 	!¨e¬lð 	!ð 	!ð 	!ð 	!ð 48Ø26ð	Qð Qà”<ðQð # 5¤<Ô0ðQð " %¤,Ô/ð	Qð
 
ŒðQð Qð Qð Qð Qð Qð Qð Qr<   r   c                   ó‚  ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 ddej        deeej        ef         de	ej                 d	e	e         d
ede	e
         de	e
         de
de	ej                 de	ej                 defˆ fd„Z	 dde	e         dd fd„Z	 	 ddede	e         dej        fd„Zdej        dej        deeej        f         fd„Zˆ xZS )ÚTokenDistillationTaskaÄ  Token-based distillation task for models with distillation heads.

    For models like DeiT that have a dedicated distillation token/head that returns
    a tuple (main_logits, dist_logits) when distilled_training is enabled. The main
    head is trained against ground truth labels while the distillation head matches
    teacher outputs.

    Supports two distillation modes:
    - 'soft': KL divergence with temperature scaling (default)
    - 'hard': Cross-entropy with teacher's hard predictions (argmax)

    Loss weighting supports two modes:
    1. Independent weights: loss = task_loss_weight * task_loss + distill_loss_weight * distill_loss
    2. Complementary mode: loss = task_loss_weight * task_loss + (1 - task_loss_weight) * distill_loss
       (used when only task_loss_weight is specified)

    Args:
        student_model: Student model with set_distilled_training() method
        teacher_model: Teacher model - can be a model name string, nn.Module, or TokenDistillationTeacher
        criterion: Task loss function for main head (default: CrossEntropyLoss)
        teacher_pretrained_path: Path to teacher pretrained weights (used when teacher_model is a string)
        distill_type: 'soft' for KL-div or 'hard' for CE with teacher argmax
        distill_loss_weight: Weight for distillation loss
        task_loss_weight: Weight for task loss
        temperature: Softmax temperature for soft distillation (ignored for hard)
        device: Device for task tensors/buffers
        dtype: Dtype for task tensors/buffers
        verbose: Enable info logging

    Example:
        >>> # With model name string (num_classes/in_chans inferred from student)
        >>> task = TokenDistillationTask(
        ...     student_model=model, teacher_model='deit_base_patch16_224',
        ...     criterion=nn.CrossEntropyLoss(),
        ...     distill_type='soft', temperature=3.0, task_loss_weight=0.5,
        ...     device=torch.device('cuda'),
        ... )
        >>> # With raw model
        >>> task = TokenDistillationTask(
        ...     student_model=model, teacher_model=my_teacher_model,
        ...     criterion=nn.CrossEntropyLoss(),
        ...     distill_type='hard', task_loss_weight=0.5,
        ... )
    NÚsoftç      ð?TÚstudent_modelÚteacher_modelÚ	criterionÚteacher_pretrained_pathÚdistill_typeÚdistill_loss_weightÚtask_loss_weightÚtemperaturer   r   Úverbosec                 óü  •— t          ¦   «                              |	|
|¬¦  «         t          |¦  «        }t          |d¦  «        st	          d|j        j        › d�¦  «        ‚|                     d¦  «         t          |t          ¦  «        r|}n�t          |t          ¦  «        st          |t          j        ¦  «        r.|j        }|j        }t          ||||| j        | j        ¬¦  «        }n$t#          dt%          |¦  «        j        › �¦  «        ‚|| _        || _        |�|nt          j        ¦   «         | _        || _        || _        |dvrt	          d	|› d
�¦  «        ‚t3          j        |j        d         | j        | j        ¬¦  «                             dddd¦  «        }t3          j        |j        d         | j        | j        ¬¦  «                             dddd¦  «        }|                      d|d¬¦  «         |                      d|d¬¦  «         |�?|| _        |�|nd| _        | j         r%tB           "                    d| j        › d|› �¦  «         n|�>|| _        d|z
  | _        | j         r%tB           "                    d|› d| j        › �¦  «         n?d| _        d| _        | j         r*tB           "                    d| j        › d| j        › �¦  «         | j         r"tB           "                    d|› d|› �¦  «         d S d S )N)r   r   rb   Úset_distilled_trainingzModel z� does not have 'set_distilled_training' method. TokenDistillationTask requires a model with a distillation head (e.g., DeiT distilled variants).T)r   r   r   r   r   r   zWteacher_model must be a model name string, nn.Module, or TokenDistillationTeacher, got )rX   ÚhardzUnsupported distill_type 'z'. Must be 'soft' or 'hard'.r   r   r   r   r   rI   Fr   rJ   rY   z9TokenDistillationTask: Independent weights - task_weight=z, distill_weight=z8TokenDistillationTask: Complementary mode - task_weight=z;TokenDistillationTask: Default equal weights - task_weight=z$TokenDistillationTask: distill_type=z, temperature=)#r#   r$   r   r1   Ú
ValueErrorr:   r.   rd   r%   r   r&   r*   r+   r   r   r   r   r,   r-   Útrainable_moduleÚteacherÚCrossEntropyLossr\   r^   ra   r3   r4   r   r5   r6   r_   r`   rb   r'   r(   )r7   rZ   r[   r\   r]   r^   r_   r`   ra   r   r   rb   Ústudent_unwrappedrh   r   r   rI   rJ   r:   s                     €r;   r$   zTokenDistillationTask.__init__³   sÙ  ø€ õ 	‰Œ×Ò ¨e¸WÐÑEÔEÐEõ )¨Ñ7Ô7ÐÝÐ(Ð*BÑCÔCð 	ÝðsÐ*Ô4Ô=ð sð sð sñô ð ð 	×0Ò0°Ñ6Ô6Ð6õ �mÕ%=Ñ>Ô>ð 	Ø#ˆGˆGÝ˜¥sÑ+Ô+ð 	­z¸-ÍÌÑ/SÔ/Sð 	à+Ô7ˆKØ(Ô1ˆHÝ.Ø%2Ø'Ø!Ø 7Ø”{Ø”jðñ ô ˆGˆGõ ð6Ý˜MÑ*Ô*Ô3ð6ð 6ñô ð ð
 !.ˆÔØˆŒØ&/Ð&;˜˜ÅÔATÑAVÔAVˆŒØ(ˆÔØ&ˆÔàÐ/Ð/Ð/ÝÐd¸,ÐdÐdÐdÑeÔeÐeõ ”|ØÔ,¨VÔ4Ø”;Ø”*ð
ñ 
ô 
÷ Š$ˆq�"�a˜Ñ
Ô
ð	 	õ
 ”lØÔ,¨UÔ3Ø”;Ø”*ð
ñ 
ô 
÷ Š$ˆq�"�a˜Ñ
Ô
ð	 	ð
 	×Ò˜^¨\ÀeÐÑLÔLÐLØ×Ò˜]¨KÀEÐÑJÔJÐJð Ð*à':ˆDÔ$Ø8HÐ8TÐ$4Ð$4ÐZ]ˆDÔ!ØŒ|ð Ý—’ðaØ#'Ô#8ðað aØK^ðað añô ð øð Ð)à$4ˆDÔ!Ø'*Ð-=Ñ'=ˆDÔ$ØŒ|ð Ý—’ðaØ#3ðað aØFJÔF^ðað añô ð øð (+ˆDÔ$Ø$'ˆDÔ!ØŒ|ð Ý—’ðfØ#'Ô#8ðfð fØKOÔKcðfð fñô ð ð
 Œ<ð 	Ý�LŠLØ`°|Ð`Ð`ÐS^Ð`Ð`ñô ð ð ð ð	ð 	r<   Ú
device_idsr@   c                 ó‚   — ddl m} | j                             ¦   «         D ]	}d|_        Œ
 || j        fd|i|¤Ž| _        | S )aŒ  Prepare task for distributed training.

        Wraps the student model in DistributedDataParallel (DDP) while leaving
        the frozen teacher model unwrapped.

        Args:
            device_ids: List of device IDs for DDP (e.g., [local_rank])
            **ddp_kwargs: Additional arguments passed to DistributedDataParallel

        Returns:
            self (for method chaining)
        r   )ÚDistributedDataParallelFrk   )Útorch.nn.parallelrm   rh   Ú
parametersÚrequires_gradrg   )r7   rk   Ú
ddp_kwargsÚDDPÚparams        r;   Úprepare_distributedz)TokenDistillationTask.prepare_distributed  sg   € ð" 	EÐDÐDÐDÐDÐDà”\×,Ò,Ñ.Ô.ð 	(ð 	(ˆEØ"'ˆEÔÐà #  DÔ$9Ð _Ð _ÀjÐ _ÐT^Ð _Ð _ˆÔØˆr<   r=   r>   r?   c                 ó�   — t          j        | j        f||dœ|¤Ž| _        | j        | _         | j        j        d||dœ|¤Ž | j        S )z=Compile student eval/train forward and teacher logit forward.rB   r"   )r3   rC   rg   Ú
eval_modelrh   rD   s       r;   rC   zTokenDistillationTask.compile5  s_   € õ !&¤¨dÔ.CÐ rÈWÐ[_Ð rÐ rÐcqÐ rÐ rˆÔàÔ/ˆŒØˆŒÔÐJ W°4ÐJÐJ¸>ÐJÐJÐJØÔ$Ð$r<   rF   Útargetc                 ó¶  — |                       |¦  «        }|\  }}|                      ||¦  «        }t          j        ¦   «         5  | j                             || j        | j        ¦  «        }|                      |                     ¦   «         ¦  «        }ddd¦  «         n# 1 swxY w Y   | j	        dk    r`t          j        || j        z  d¬¦  «        }	t          j        || j        z  d¬¦  «        }
t          j        |	|
dd¬¦  «        | j        dz  z  }n+|                     d¬¦  «        }t          j        ||¦  «        }| j        |z  | j        |z  z   }||||d	œS )
aÂ  Forward pass with token distillation.

        Args:
            input: Input tensor [B, C, H, W]
            target: Target labels [B]

        Returns:
            Dictionary containing:
                - 'loss': Combined training loss (task + distillation)
                - 'output': Main head logits (for metrics)
                - 'task_loss': Classification loss component
                - 'distill_loss': Distillation loss component
        NrX   r   )ÚdimÚ	batchmeanT)Ú	reductionÚ
log_targeté   )ÚlossÚoutputÚ	task_lossÚdistill_loss)rg   r\   r3   Úno_gradrh   rM   rI   rJ   Údetachr^   ÚFÚlog_softmaxra   Úkl_divÚargmaxÚcross_entropyr`   r_   )r7   rF   rw   Ústudent_outputÚmain_logitsÚdist_logitsr€   Úinput_kdÚteacher_logitsÚprob_sÚprob_tr�   Úteacher_hardÚ
total_losss                 r;   rH   zTokenDistillationTask.forwardB  s¤  € ð& ×.Ò.¨uÑ5Ô5ˆØ#1Ñ ˆ�[ð —N’N ;°Ñ7Ô7ˆ	õ Œ]‰_Œ_ð 	=ð 	=Ø”|×3Ò3°E¸4Ô;LÈdÔN^Ñ_Ô_ˆHØ!Ÿ\š\¨(¯/ª/Ñ*;Ô*;Ñ<Ô<ˆNð	=ð 	=ð 	=ñ 	=ô 	=ð 	=ð 	=ð 	=ð 	=ð 	=ð 	=øøøð 	=ð 	=ð 	=ð 	=ð
 Ô Ò&Ð&Ý”] ;°Ô1AÑ#AÀrÐJÑJÔJˆFÝ”] >°DÔ4DÑ#DÈ"ÐMÑMÔMˆFÝœ8 F¨F¸kÐVZÐ[Ñ[Ô[Ð_cÔ_oÐstÑ_tÑuˆLˆLà)×0Ò0°RÐ0Ñ8Ô8ˆLÝœ?¨;¸ÑEÔEˆLàÔ*¨YÑ6¸Ô9QÐT`Ñ9`Ñ`ˆ
ð Ø!Ø"Ø(ð	
ð 
ð 	
s   ÁABÂB"Â%B")	NNrX   NNrY   NNT)NrN   )r.   rO   rP   rQ   r*   r+   r   r&   r   r   Úfloatr3   r   r   Úboolr$   Úlistrt   rC   rS   r   rH   rT   rU   s   @r;   rW   rW   …   sÒ  ø€ € € € € ð+ð +ðb .2Ø59Ø &Ø37Ø04Ø!$Ø-1Ø+/Ø ðgð gàœ9ðgð !  b¤iÐ1IÐ!IÔJðgð   ¤	Ô*ð	gð
 &.¨c¤]ðgð ðgð "*¨%¤ðgð ' uœoðgð ðgð ˜Uœ\Ô*ðgð ˜EœKÔ(ðgð ðgð gð gð gð gð gðV *.ðð à  œðð 
!ð	ð ð ð ð6 &Ø"&ð%ð %àð%ð ˜3”-ð%ð
 
Œð%ð %ð %ð %ð.
à”<ð.
ð ”Lð.
ð 
ˆc�5”<ÐÔ	 ð	.
ð .
ð .
ð .
ð .
ð .
ð .
ð .
r<   rW   )rQ   ÚloggingÚtypingr   r   r   r3   Útorch.nnr*   Útorch.nn.functionalÚ
functionalr„   Útimm.modelsr   Ú
timm.utilsr   Útaskr	   Ú	getLoggerr.   r'   r+   r   rW   r"   r<   r;   ú<module>rž      s+  ðØ PÐ PØ €€€Ø (Ð (Ð (Ð (Ð (Ð (Ð (Ð (Ð (Ð (à €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð à $Ð $Ð $Ð $Ð $Ð $Ø #Ð #Ð #Ð #Ð #Ð #à Ð Ð Ð Ð Ð à
ˆ'Ô
˜HÑ
%Ô
%€ðqQð qQð qQð qQð qQ˜rœyñ qQô qQð qQðhk
ð k
ð k
ð k
ð k
˜Lñ k
ô k
ð k
ð k
ð k
r<   