§
    9²¸j”q  ã                   óÒ  — d dl mZ d dlZd dlmZ ddlmZ ddlmZ ddl	m
Z
mZ ddlmZ dd	lmZmZmZ dd
lmZmZ ddlmZ ddlmZmZ ddlmZmZ ddlmZmZ ddl m!Z! ddl"m#Z#m$Z$m%Z% ddl&m'Z' ddl(m)Z)m*Z* ddl+m,Z,m-Z- ddl.m/Z/  G d„ dej0        ¦  «        Z1 ed¦  «         G d„ dej0        ¦  «        ¦   «         Z2 G d„ dej0        ¦  «        Z3 G d„ dej0        ¦  «        Z4e G d„ d ej0        ¦  «        ¦   «         Z5 G d!„ d"ej0        ¦  «        Z6d#„ Z7 ed$¦  «        d?d%„¦   «         Z8d&ej9        d'e:d(ej9        fd)„Z;	 d@d+ej0        d,ej9        d-ej9        d.ej9        d/ej9        dz  d0e<d1e<d2e!e#         fd3„Z= ee8¦  «         G d4„ d5ej0        ¦  «        ¦   «         Z> G d6„ d7e¦  «        Z? G d8„ d9e¦  «        Z@e$ G d:„ d;e@¦  «        ¦   «         ZAe$ G d<„ d=e@e¦  «        ¦   «         ZBg d>¢ZCdS )Aé    )ÚCallableN)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_experts_implementationÚuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚAfmoeConfigc            	       óî   ‡ — e Zd Z edd¬¦  «        d
defˆ fd„¦   «         Ze edd¬¦  «        d
dedeej	        e
f         fd„¦   «         ¦   «         Z ej        ¦   «         ed	„ ¦   «         ¦   «         Zˆ xZS )ÚAfmoeRotaryEmbeddingÚdevicez5.18)ÚversionNÚconfigc                 ó¾  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        | j        j        d         | _        | j        }| j        dk    rt          | j                 } || j        |¦  «        \  }| _
        t          j        |d¬¦  «        | _        t          j        |                     ¦   «         d¬¦  «        | _        d S )NÚ	rope_typeÚdefaultF)Ú
persistent)ÚsuperÚ__init__Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr&   Úrope_parametersr(   Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)Úselfr&   r$   Úrope_init_fnr4   Ú	__class__s        €úx/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/afmoe/modeling_afmoe.pyr,   zAfmoeRotaryEmbedding.__init__-   sÀ   ø€ å‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!àˆŒàœÔ4°[ÔAˆŒØ!%Ô!EˆØŒ>˜YÒ&Ð&Ý.¨t¬~Ô>ˆLØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(åœ	 (°uÐ=Ñ=Ô=ˆŒÝ!#¤¨8¯>ª>Ñ+;Ô+;ÈÐ!NÑ!NÔ!NˆÔÐÐó    Úreturnc                 óä   — | j         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «        |z  z  z  }|                     |¦  «        |fS )a¾  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNg      ð?r   é   ©Údtype)r0   ÚgetattrÚhidden_sizeÚnum_attention_headsÚtorchÚarangeÚfloatÚto)r&   r$   ÚkwargsÚbaseÚdimÚattention_factorr4   s          r:   r1   z4AfmoeRotaryEmbedding.compute_default_rope_parameters>   sz   € ð Ô% lÔ3ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐà˜$¥5¤<°°3¸ÅÄÐ#MÑ#MÔ#MÐPSÑ#SÑTÑUˆØ�{Š{˜6Ñ"Ô"Ð$4Ð4Ð4r;   c                 óú  — | j         d d d …d f                              |j        d         dd¦  «                             t          j        |j        ¬¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  ||z                       dd	¦  «        }t	          j        ||fd¬
¦  «        }|                     ¦   «         | j        z  }|                     ¦   «         | j        z  }	d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬¦  «        |	                     |j        ¬¦  «        fS )Nr   éÿÿÿÿr    )rB   r$   ÚmpsÚcpuF)Údevice_typeÚenabledr@   ©rL   rA   )r4   ÚexpandÚshaperI   rF   rH   r$   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosr2   ÚsinrB   )
r7   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedrR   ÚfreqsÚembr\   r]   s
             r:   ÚforwardzAfmoeRotaryEmbedding.forwardR   s›  € ð ŒM˜$    4˜-Ô(×/Ò/°Ô0BÀ1Ô0EÀrÈ1ÑMÔM×PÒPÕW\ÔWbÐklÔksÐPÑtÔtð 	ð !-¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆå¨¸UÐCÑCÔCð 	5ð 	5Ø&Ð)>Ñ>×IÒIÈ!ÈQÑOÔOˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCð		5ð 	5ð 	5ñ 	5ô 	5ð 	5ð 	5ð 	5ð 	5ð 	5ð 	5øøøð 	5ð 	5ð 	5ð 	5ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   ÃA*D<Ä<E ÅE ©N)Ú__name__Ú
__module__Ú__qualname__r   r!   r,   ÚstaticmethodÚtuplerF   ÚTensorrH   r1   Úno_gradr   rd   Ú__classcell__©r9   s   @r:   r#   r#   ,   sö   ø€ € € € € Ø€_�X vÐ.Ñ.Ô.ðOð O˜{ð Oð Oð Oð Oð Oñ /Ô.ðOð  Ø€_�X vÐ.Ñ.Ô.ð5ð 5°ð 5ÐW\Ð]bÔ]iÐkpÐ]pÔWqð 5ð 5ð 5ñ /Ô.ñ „\ð5ð$ €U„]�_„_Øð<ð <ñ Ôñ „_ð<ð <ð <ð <ð <r;   r#   ÚRMSNormc                   óF   ‡ — e Zd Zddeddfˆ fd„Zdej        fd„Zd„ Zˆ xZ	S )	ÚAfmoeRMSNormç�íµ ÷Æ°>Úepsr<   Nc                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )z;
        AfmoeRMSNorm is equivalent to T5LayerNorm
        N)r+   r,   r   Ú	ParameterrF   ÚonesÚweightÚvariance_epsilon)r7   rD   rs   r9   s      €r:   r,   zAfmoeRMSNorm.__init__g   sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐr;   c                 ó  — |j         }|                     t          j        ¦  «        }|                     d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        |z                       |¦  «        S )Nr@   rO   T)Úkeepdim)	rB   rI   rF   Úfloat32ÚpowÚmeanÚrsqrtrx   rw   )r7   Úhidden_statesÚinput_dtypeÚvariances       r:   rd   zAfmoeRMSNorm.forwardo   s|   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆØ”˜mÑ+×/Ò/°Ñ<Ô<Ð<r;   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)rj   rw   rV   rx   )r7   s    r:   Ú
extra_reprzAfmoeRMSNorm.extra_reprv   s&   € Ý˜œÔ)Ñ*Ô*ÐIÐI°$Ô2GÐIÐIÐIr;   )rr   )
rf   rg   rh   rH   r,   rF   rk   rd   rƒ   rm   rn   s   @r:   rq   rq   e   sƒ   ø€ € € € € ð$ð $¨ð $¸$ð $ð $ð $ð $ð $ð $ð=¨¬ð =ð =ð =ð =ðJð Jð Jð Jð Jð Jð Jr;   rq   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚAfmoeMLPNc                 ó   •— t          ¦   «                              ¦   «          || _        |j        | _        |€|j        n|| _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _	        t          |j                 | _        d S ©NF©Úbias)r+   r,   r&   rD   Úintermediate_sizer   ÚLinearÚ	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fn)r7   r&   rŠ   r9   s      €r:   r,   zAfmoeMLP.__init__{   s±   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ=NÐ=V Ô!9Ð!9Ð\mˆÔÝœ 4Ô#3°TÔ5KÐRWÐXÑXÔXˆŒÝ”y Ô!1°4Ô3IÐPUÐVÑVÔVˆŒÝœ 4Ô#9¸4Ô;KÐRWÐXÑXÔXˆŒÝ˜VÔ.Ô/ˆŒˆˆr;   c                 ó¨   — |                       |                      |                      |¦  «        ¦  «        |                      |¦  «        z  ¦  «        }|S re   )rŽ   r�   rŒ   r�   )r7   r^   rŽ   s      r:   rd   zAfmoeMLP.forward…   sA   € Ø—N’N 4§;¢;¨t¯~ª~¸aÑ/@Ô/@Ñ#AÔ#AÀDÇLÂLÐQRÁOÄOÑ#SÑTÔTˆ	ØÐr;   re   )rf   rg   rh   r,   rd   rm   rn   s   @r:   r…   r…   z   sL   ø€ € € € € ð0ð 0ð 0ð 0ð 0ð 0ðð ð ð ð ð ð r;   r…   c                   óF   ‡ — e Zd ZdZˆ fd„Zdej        dej        fd„Zˆ xZS )ÚAfmoeTokenChoiceRouterz©
    Token-choice top-K router for MoE routing.

    This router assigns each token to the top-K experts based on sigmoid scores, matching the released checkpoints.
    c                 óê   •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        |j        | _        t          j        |j	        |j        d¬¦  «        | _
        d S r‡   )r+   r,   r&   Únum_experts_per_tokÚtop_kÚnum_expertsÚroute_scaler   r‹   rD   Úgate©r7   r&   r9   s     €r:   r,   zAfmoeTokenChoiceRouter.__init__‘   sb   ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ/ˆŒ
Ø!Ô-ˆÔØ!Ô-ˆÔÝ”I˜fÔ0°&Ô2DÈ5ÐQÑQÔQˆŒ	ˆ	ˆ	r;   r   Úexpert_biasc                 óž  — |j         \  }}}|                     d|¦  «        }|                      |¦  «                             t          j        ¦  «        }t	          j        |¦  «        }t	          j        ||z   | j        d¬¦  «        \  }}| 	                    d|¬¦  «        }| 
                    dd¬¦  «        dz   }	||	z  }|| j        z  }|||fS )NrO   r    )ÚkrL   )rL   ÚindexT)rL   rz   g#B’¡œÇ;)rV   Úviewr™   rI   rF   r{   ÚsigmoidÚtopkr–   ÚgatherÚsumr˜   )
r7   r   r›   Ú_Ú
hidden_dimÚrouter_logitsÚscoresÚselected_expertsÚ
top_scoresÚdenominators
             r:   rd   zAfmoeTokenChoiceRouter.forward™   sÏ   € Ø(Ô.Ñˆˆ1ˆjØ%×*Ò*¨2¨zÑ:Ô:ˆàŸ	š	 -Ñ0Ô0×3Ò3µE´MÑBÔBˆÝ”˜}Ñ-Ô-ˆå#œj¨°+Ñ)=ÀÄÐQRÐSÑSÔSÑˆÐØ—]’] qÐ0@�]ÑAÔAˆ
Ø —n’n¨°T�nÑ:Ô:¸UÑBˆØ +Ñ-ˆ
Ø $Ô"2Ñ2ˆ
Ø˜jÐ*:Ð:Ð:r;   ©	rf   rg   rh   Ú__doc__r,   rF   rk   rd   rm   rn   s   @r:   r“   r“   Š   sm   ø€ € € € € ðð ðRð Rð Rð Rð Rð; U¤\ð ;ÀÄð ;ð ;ð ;ð ;ð ;ð ;ð ;ð ;r;   r“   c                   ób   ‡ — e Zd ZdZˆ fd„Zdej        dej        dej        dej        fd„Zˆ xZS )ÚAfmoeExpertsz2Collection of expert weights stored as 3D tensors.c                 ó´  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        t          j        t          j
        | j        d| j        z  | j        ¦  «        ¦  «        | _        t          j        t          j
        | j        | j        | j        ¦  «        ¦  «        | _        t          |j                 | _        d S )Nr@   )r+   r,   r—   rD   r¥   Úmoe_intermediate_sizeÚintermediate_dimr   ru   rF   ÚemptyÚgate_up_projrŽ   r   r�   r�   rš   s     €r:   r,   zAfmoeExperts.__init__¬   s£   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔØ Ô,ˆŒØ &Ô <ˆÔÝœL­¬°TÔ5EÀqÈ4ÔK`ÑG`ÐbfÔbqÑ)rÔ)rÑsÔsˆÔÝœ¥e¤k°$Ô2BÀDÄOÐUYÔUjÑ&kÔ&kÑlÔlˆŒÝ˜VÔ.Ô/ˆŒˆˆr;   r   Útop_k_indexÚtop_k_weightsr<   c                 ó€  — t          j        |¦  «        }t          j        ¦   «         5  t           j        j                             || j        ¬¦  «        }|                     ddd¦  «        }t          j        | 	                    d¬¦  «        d¦  «         
                    ¦   «         }d d d ¦  «         n# 1 swxY w Y   |D ]þ}|d         }|| j        k    rŒt          j        ||         ¦  «        \  }}	||	         }
t          j                             |
| j        |         ¦  «                             dd¬¦  «        \  }}|                      |¦  «        |z  }t          j                             || j        |         ¦  «        }|||	|d f         z  }|                     d|	|                     |j        ¦  «        ¦  «         Œÿ|S )N)Únum_classesr@   r    r   )rO   éþÿÿÿrT   rO   )rF   Ú
zeros_likerl   r   Ú
functionalÚone_hotr—   ÚpermuteÚgreaterr£   ÚnonzeroÚwhereÚlinearr³   Úchunkr�   rŽ   Ú
index_add_rI   rB   )r7   r   r´   rµ   Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stater™   ÚupÚcurrent_hidden_statess                 r:   rd   zAfmoeExperts.forwardµ   sø  € õ $Ô.¨}Ñ=Ô=ÐÝŒ]‰_Œ_ð 	Sð 	SÝœ(Ô-×5Ò5°kÈtÔO_Ð5Ñ`Ô`ˆKØ%×-Ò-¨a°°AÑ6Ô6ˆKÝœ {§¢¸8 Ñ'DÔ'DÀaÑHÔH×PÒPÑRÔRˆJð	Sð 	Sð 	Sñ 	Sô 	Sð 	Sð 	Sð 	Sð 	Sð 	Sð 	Søøøð 	Sð 	Sð 	Sð 	Sð
 %ð 
	nð 
	nˆJØ# AœˆJØ˜TÔ-Ò-Ð-ØÝ#(¤;¨{¸:Ô/FÑ#GÔ#GÑ ˆI�yØ)¨)Ô4ˆMÝ”}×+Ò+¨M¸4Ô;LÈZÔ;XÑYÔY×_Ò_Ð`aÐgiÐ_ÑjÔj‰HˆD�"Ø$(§K¢K°Ñ$5Ô$5¸Ñ$:Ð!Ý$&¤M×$8Ò$8Ð9NÐPTÔP^Ð_iÔPjÑ$kÔ$kÐ!Ø$9¸MÈ)ÐU^Ð`dÐJdÔ<eÑ$eÐ!Ø×*Ò*¨1¨iÐ9N×9QÒ9QÐReÔRkÑ9lÔ9lÑmÔmÐmÐmà"Ð"s   ¨A>B2Â2B6Â9B6r«   rn   s   @r:   r®   r®   ¨   s€   ø€ € € € € à<Ð<ð0ð 0ð 0ð 0ð 0ð#à”|ð#ð ”\ð#ð ”|ð	#ð
 
Œð#ð #ð #ð #ð #ð #ð #ð #r;   r®   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚAfmoeSparseMoeBlockzÎ
    Mixture of Experts (MoE) module for AFMoE.

    This module implements a sparse MoE layer with both shared experts (always active) and
    routed experts (activated based on token-choice routing).
    c                 óN  •— t          ¦   «                              ¦   «          || _        t          |¦  «        | _        t          ||j        |j        z  ¦  «        | _        t          |¦  «        | _
        t          j        t          j        |j        ¦  «        d¬¦  «        | _        d S )NF)Úrequires_grad)r+   r,   r&   r“   Úrouterr…   r°   Únum_shared_expertsÚshared_expertsr®   Úexpertsr   ru   rF   Úzerosr—   r›   rš   s     €r:   r,   zAfmoeSparseMoeBlock.__init__Ø   s†   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ,¨VÑ4Ô4ˆŒÝ& v¨vÔ/KÈfÔNgÑ/gÑhÔhˆÔÝ# FÑ+Ô+ˆŒÝœ<­¬°FÔ4FÑ(GÔ(GÐW\Ð]Ñ]Ô]ˆÔÐÐr;   c                 ó8  — |j         \  }}}|                     d|¦  «        }|                      || j        ¦  «        \  }}}|                      |¦  «                             |||¦  «        }	|                      |||¦  «                             |||¦  «        }
|	|
z   S )NrO   )rV   rŸ   rÐ   r›   rÒ   rÓ   )r7   r   Ú
batch_sizeÚseq_lenr¥   Úhidden_states_flatr¤   r©   r¨   Úshared_outputÚrouted_outputs              r:   rd   zAfmoeSparseMoeBlock.forwardà   s¬   € Ø*7Ô*=Ñ'ˆ
�G˜ZØ*×/Ò/°°JÑ?Ô?Ðð +/¯+ª+°mÀTÔEUÑ*VÔ*VÑ'ˆˆ:Ð'ð ×+Ò+Ð,>Ñ?Ô?×DÒDÀZÐQXÐZdÑeÔeˆØŸšÐ%7Ð9IÈ:ÑVÔV×[Ò[Ø˜ ñ
ô 
ˆð ˜}Ñ,Ð,r;   )rf   rg   rh   r¬   r,   rd   rm   rn   s   @r:   rÍ   rÍ   Ð   sV   ø€ € € € € ðð ð^ð ^ð ^ð ^ð ^ð-ð -ð -ð -ð -ð -ð -r;   rÍ   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..NrO   r@   rT   )rV   rF   r[   )r^   Úx1Úx2s      r:   Úrotate_halfrÞ   ï   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'r;   Úrotary_pos_embc                 ó¾   — |                      |¦  «        }|                      |¦  «        }| |z  t          | ¦  «        |z  z   }||z  t          |¦  «        |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezerÞ   )Úqr�   r\   r]   Úunsqueeze_dimÚq_embedÚk_embeds          r:   Úapply_rotary_pos_embræ   ö   sc   € ð& �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�GÐÐr;   r   Ún_repr<   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r    N)rV   rU   Úreshape)r   rç   ÚbatchÚnum_key_value_headsÚslenr?   s         r:   Ú	repeat_kvrí     s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr;   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutrJ   c                 ó  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
dt          j        ¬¦  «         	                    |j
        ¦  «        }
t
          j                             |
|| j        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )Nr@   r   rO   )rL   rB   )ÚpÚtrainingr    )rí   Únum_key_value_groupsrF   ÚmatmulrZ   r   rº   Úsoftmaxr{   rI   rB   rõ   rø   Ú
contiguous)rï   rð   rñ   rò   ró   rô   rõ   rJ   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r:   Úeager_attention_forwardr    sé   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r;   c                   óÄ   ‡ — e Zd ZdZdedefˆ fd„Z	 ddej        de	ej        ej        f         dej        dz  d	e
dz  d
ee         de	ej        ej        f         fd„Zˆ xZS )ÚAfmoeAttentionaJ  
    Multi-headed attention module with optional sliding window and gating.

    This attention mechanism supports both full attention and sliding window attention,
    and includes Q/K normalization and gating of the output. It inherits from [`LlamaAttention`] to minimize the amount
    of custom logic we need to maintain.
    r&   Ú	layer_idxc                 óà  •— t          ¦   «                              ¦   «          || _        || _        t	          |d|j        |j        z  ¦  «        | _        |j        |j        z  | _	        | j        dz  | _
        |j        | _        d| _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        | j        z  |j        |j        ¬¦  «        | _        |j        |         dk    | _        | j        r|j        nd | _        t/          | j        |j        ¬¦  «        | _        t/          | j        |j        ¬¦  «        | _        t          j        |j        |j        | j        z  d¬¦  «        | _        d S )Nr?   g      à¿Trˆ   Úsliding_attention©rs   F)r+   r,   r&   r  rC   rD   rE   r?   rë   rù   rô   Úattention_dropoutÚ	is_causalr   r‹   Úattention_biasÚq_projÚk_projÚv_projÚo_projÚlayer_typesÚis_local_attentionÚsliding_windowrq   Úrms_norm_epsÚq_normÚk_normrŒ   ©r7   r&   r  r9   s      €r:   r,   zAfmoeAttention.__init__?  sÍ  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒÝ ¨
°FÔ4FÈ&ÔJdÑ4dÑeÔeˆŒØ$*Ô$>À&ÔB\Ñ$\ˆÔ!Ø”} dÑ*ˆŒØ!'Ô!9ˆÔØˆŒå”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ&¨¬Ñ6¸Ô8JÐQWÔQfð
ñ 
ô 
ˆŒð
 #)Ô"4°YÔ"?ÐCVÒ"VˆÔØ7;Ô7NÐX˜fÔ3Ð3ÐTXˆÔå" 4¤=°fÔ6IÐJÑJÔJˆŒÝ" 4¤=°fÔ6IÐJÑJÔJˆŒÝœ 6Ô#5°vÔ7QÐTXÔTaÑ7aÐhmÐnÑnÔnˆŒˆˆr;   Nr   Úposition_embeddingsró   Úpast_key_valuesrJ   r<   c                 óì  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «        }|                      |¦  «                             |¦  «        }	|                      |¦  «                             |¦  «        }
|                      |¦  «        }|                      |¦  «                             dd¦  «        }|  	                    |	¦  «                             dd¦  «        }	|
                     dd¦  «        }
| j
        r|\  }}t          ||	||¦  «        \  }}	|�|                     |	|
| j        ¦  «        \  }	}
t          j        | j        j        t$          ¦  «        } || ||	|
f|| j        sdn| j        | j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|t1          j        |¦  «        z  }|                      |¦  «        }||fS )NrO   r    r@   rî   )ró   rõ   rô   r  )rV   r?   r  rŸ   r  r  rŒ   r  rZ   r  r  ræ   Úupdater  r   Úget_interfacer&   Ú_attn_implementationr  rø   r  rô   r  rü   rF   r    r  )r7   r   r  ró   r  rJ   Úinput_shapeÚhidden_shapeÚquery_statesrý   rþ   Úgate_statesr\   r]   Úattention_interfaceÚoutputrÿ   r   s                     r:   rd   zAfmoeAttention.forward^  s  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔDˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔBˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔDˆØ—n’n ]Ñ3Ô3ˆà—{’{ <Ñ0Ô0×:Ò:¸1¸aÑ@Ô@ˆØ—[’[ Ñ,Ô,×6Ò6°q¸!Ñ<Ô<ˆ
Ø#×-Ò-¨a°Ñ3Ô3ˆàÔ"ð 	`Ø*‰HˆC�Ý';¸LÈ*ÐVYÐ[^Ñ'_Ô'_Ñ$ˆL˜*àÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð  3Ð2ØØØØð	
 
ð
 *Ø#œ}ÐH�C�C°$Ô2HØ”LØÔ.ð
 
ð 
 
ð ð
 
ð 
 
Ñˆ�ð �”Ð.˜kÐ.¨2Ð.Ð.Ð.×9Ò9Ñ;Ô;ˆØ�%œ-¨Ñ4Ô4Ñ4ˆØ—k’k &Ñ)Ô)ˆØ˜LÐ(Ð(r;   re   )rf   rg   rh   r¬   r!   Úintr,   rF   rk   rj   r   r   r   rd   rm   rn   s   @r:   r  r  5  så   ø€ € € € € ðð ðo˜{ð o°sð oð oð oð oð oð oðH )-ð.)ð .)à”|ð.)ð # 5¤<°´Ð#=Ô>ð.)ð œ tÑ+ð	.)ð
  ™ð.)ð Ð+Ô,ð.)ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð.)ð .)ð .)ð .)ð .)ð .)ð .)ð .)r;   r  c                   óÖ   ‡ — e Zd ZdZdedefˆ fd„Z	 	 	 	 	 ddej        dej        dz  dej	        dz  d	e
dz  d
edz  deej        ej        f         dz  dee         dej        fd„Zˆ xZS )ÚAfmoeDecoderLayerzÌ
    AFMoE decoder layer with dual normalization.

    This layer applies self-attention followed by either a dense MLP or MoE block,
    with dual normalization (pre and post) around each component.
    r&   r  c                 ó  •— t          ¦   «                              ¦   «          |j        | _        || _        t	          ||¬¦  «        | _        t          |j        |j        ¬¦  «        | _        t          |j        |j        ¬¦  «        | _	        t          |j        |j        ¬¦  «        | _
        t          |j        |j        ¬¦  «        | _        ||j        k    | _        | j        rt          |¦  «        | _        d S t!          |¦  «        | _        d S )N)r&   r  r  )r+   r,   rD   r  r  Ú	self_attnrq   r  Úinput_layernormÚpost_attention_layernormÚpre_mlp_layernormÚpost_mlp_layernormÚnum_dense_layersÚmoe_enabledrÍ   Úmlpr…   r  s      €r:   r,   zAfmoeDecoderLayer.__init__—  sò   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔØ"ˆŒå'¨vÀÐKÑKÔKˆŒõ  ,¨FÔ,>ÀFÔDWÐXÑXÔXˆÔÝ(4°VÔ5GÈVÔM`Ð(aÑ(aÔ(aˆÔ%õ ".¨fÔ.@ÀfÔFYÐ!ZÑ!ZÔ!ZˆÔÝ".¨vÔ/AÀvÔGZÐ"[Ñ"[Ô"[ˆÔð %¨Ô(?Ò?ˆÔØÔð 	(Ý*¨6Ñ2Ô2ˆDŒHˆHˆHå Ñ'Ô'ˆDŒHˆHˆHr;   Nr   ró   r_   r  Ú	use_cacher  rJ   r<   c           
      ó"  — |}|                       |¦  «        } | j        d||||||dœ|¤Ž\  }}	|                      |¦  «        }||z   }|}|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }||z   }|S )N)r   ró   r_   r  r.  r  © )r'  r&  r(  r)  r-  r*  )
r7   r   ró   r_   r  r.  r  rJ   Úresidualr¤   s
             r:   rd   zAfmoeDecoderLayer.forward­  sÉ   € ð !ˆð ×,Ò,¨]Ñ;Ô;ˆØ)˜4œ>ð 
Ø'Ø)Ø%Ø+ØØ 3ð
ð 
ð ð
ð 
Ñˆ�qð ×5Ò5°mÑDÔDˆØ  =Ñ0ˆð !ˆØ×.Ò.¨}Ñ=Ô=ˆØŸš Ñ/Ô/ˆØ×/Ò/°Ñ>Ô>ˆà  =Ñ0ˆØÐr;   )NNNNN)rf   rg   rh   r¬   r!   r"  r,   rF   rk   Ú
LongTensorr   Úboolrj   r   r   ÚFloatTensorrd   rm   rn   s   @r:   r$  r$  �  s  ø€ € € € € ðð ð(˜{ð (°sð (ð (ð (ð (ð (ð (ð2 /3Ø04Ø(,Ø!%ØHLð!ð !à”|ð!ð œ tÑ+ð!ð Ô&¨Ñ-ð	!ð
  ™ð!ð ˜$‘;ð!ð # 5¤<°´Ð#=Ô>ÀÑEð!ð Ð+Ô,ð!ð 
Ô	ð!ð !ð !ð !ð !ð !ð !ð !r;   r$  c                   ó€   ‡ — e Zd ZU dZeed<   dZdgZdgZ e	e
d¬¦  «        eedœZg d	¢Zd
Zd
Zd
Zd
Zd
Zd
Zˆ fd„Zˆ xZS )ÚAfmoePreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    r&   Úmodelr$  r  r   )rž   )r¦   r   Ú
attentions)r'  r(  r)  r*  r  r  Únormr›   Tc                 óÊ  •— t          ¦   «                              |¦  «         | j        j        }t	          |t
          ¦  «        r:t          j        |j        d|¬¦  «         t          j        |j	        d|¬¦  «         dS t	          |t          ¦  «        r t          j        |j        j        ¦  «         dS t	          |t          ¦  «        rt          j        |j        ¦  «         dS dS )zInitialize the weightsrî   )r}   ÚstdN)r+   Ú_init_weightsr&   Úinitializer_rangerW   r®   ÚinitÚnormal_r³   rŽ   r“   Úzeros_r™   rw   rÍ   r›   )r7   rï   r;  r9   s      €r:   r<  z"AfmoePreTrainedModel._init_weightsñ  sÕ   ø€ å‰Œ×Ò˜fÑ%Ô%Ð%ØŒkÔ+ˆÝ�f�lÑ+Ô+ð 	,ÝŒL˜Ô,°3¸CÐ@Ñ@Ô@Ð@ÝŒL˜Ô)°¸Ð=Ñ=Ô=Ð=Ð=Ð=Ý˜Õ 6Ñ7Ô7ð 	,ÝŒK˜œÔ*Ñ+Ô+Ð+Ð+Ð+Ý˜Õ 3Ñ4Ô4ð 	,ÝŒK˜Ô*Ñ+Ô+Ð+Ð+Ð+ð	,ð 	,r;   )rf   rg   rh   r¬   r!   Ú__annotations__Úbase_model_prefixÚ_no_split_modulesÚ_skip_keys_device_placementr   r“   r$  r  Ú_can_record_outputsÚ_keep_in_fp32_modulesÚ_supports_sdpaÚ_supports_flash_attnÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendÚsupports_gradient_checkpointingr<  rm   rn   s   @r:   r6  r6  Ñ  sÇ   ø€ € € € € € ðð ð
 ÐÐÑØÐØ,Ð-ÐØ#4Ð"5Ðà'˜Ð(>ÀaÐHÑHÔHØ*Ø$ðð Ðð
	ð 	ð 	Ðð €NØÐØÐØ!ÐØ"&ÐØ&*Ð#ð
,ð 
,ð 
,ð 
,ð 
,ð 
,ð 
,ð 
,ð 
,r;   r6  c                   óì   ‡ — e Zd ZdZdefˆ fd„Zeee	 	 	 	 	 	 dde	j
        dz  de	j        dz  de	j        dz  de	j
        dz  d	edz  d
edz  dee         deez  fd„¦   «         ¦   «         ¦   «         Zˆ xZS )Ú
AfmoeModelz›
    Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`AfmoeDecoderLayer`]

    Args:
        config: AfmoeConfig
    r&   c                 óÞ  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          j        ‰j        ‰j        | j        ¦  «        | _        t          j	        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t!          ‰¬¦  «        | _        d| _        |                      ¦   «          d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r0  )r$  )Ú.0r  r&   s     €r:   ú
<listcomp>z'AfmoeModel.__init__.<locals>.<listcomp>  s$   ø€ ÐcÐcÐc°iÕ˜v yÑ1Ô1ÐcÐcÐcr;   r  ©r&   F)r+   r,   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	EmbeddingrD   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersÚlayersrq   r  r9  r#   Ú
rotary_embÚgradient_checkpointingÚ	post_initrš   s    `€r:   r,   zAfmoeModel.__init__  sÒ   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒåœL¨Ô):¸FÔ<NÐPTÔP`ÑaÔaˆÔÝ”mØcÐcÐcÐcÅ5ÈÔIaÑCbÔCbÐcÑcÔcñ
ô 
ˆŒõ ! Ô!3¸Ô9LÐMÑMÔMˆŒ	Ý.°fÐ=Ñ=Ô=ˆŒØ&+ˆÔ#à�ŠÑÔÐÐÐr;   NÚ	input_idsró   Úinputs_embedsr_   r  r.  rJ   r<   c           
      óø  — |d u |d uz  rt          d¦  «        ‚|r|€t          | j        ¬¦  «        }|€|                      |¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }t          |x}	t          ¦  «        s%| j        |||dœ}
t          di |
¤Žt          di |
¤Ždœ}	|}| j        j        r|| j        j        dz  z  }|                      ||¦  «        }t#          | j        ¦  «        D ]*\  }} ||f|	| j        j        |                  ||||d	œ|¤Ž}Œ+|                      |¦  «        }t+          ||r|nd ¬
¦  «        S )Nz:You must specify exactly one of input_ids or inputs_embedsrS  r   r    )r$   )r&   ra  ró   r  )Úfull_attentionr  g      à?)ró   r_   r  r.  r  )Úlast_hidden_stater  r0  )Ú
ValueErrorr	   r&   rX  Úget_seq_lengthrF   rG   rV   r$   rá   rW   Údictr   r   Úmup_enabledrD   r]  Ú	enumerater\  r  r9  r   )r7   r`  ró   ra  r_   r  r.  rJ   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsr   r  ÚiÚdecoder_layers                  r:   rd   zAfmoeModel.forward  s  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLõ °Ð?Ð-ÅÑFÔFð 
	àœ+Ø!.Ø"0Ø#2ð	ð ˆKõ #5Ð"CÐ"C°{Ð"CÐ"CÝ%FÐ%UÐ%UÈÐ%UÐ%Uð#ð #Ðð
 &ˆð Œ;Ô"ð 	KØ)¨T¬[Ô-DÀcÑ-IÑJˆMà"Ÿošo¨m¸\ÑJÔJÐå )¨$¬+Ñ 6Ô 6ð 		ð 		ÑˆAˆ}Ø)˜MØðà2°4´;Ô3JÈ1Ô3MÔNØ)Ø /Ø#Ø$7ðð ð ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆÝ%Ø+Ø/8ÐB˜O˜O¸dð
ñ 
ô 
ð 	
r;   )NNNNNN)rf   rg   rh   r¬   r!   r,   r   r   r   rF   r2  rk   r4  r   r3  r   r   rj   r   rd   rm   rn   s   @r:   rN  rN  þ  s  ø€ € € € € ðð ð˜{ð ð ð ð ð ð ð ØØð .2Ø.2Ø26Ø04Ø(,Ø!%ð<
ð <
àÔ# dÑ*ð<
ð œ tÑ+ð<
ð Ô(¨4Ñ/ð	<
ð
 Ô&¨Ñ-ð<
ð  ™ð<
ð ˜$‘;ð<
ð Ð+Ô,ð<
ð 
Ð'Ñ	'ð<
ð <
ð <
ñ „_ñ  Ôñ „^ð<
ð <
ð <
ð <
ð <
r;   rN  c                   ó,  ‡ — e Zd ZddiZddiZddgdgfiZddiZˆ fd„Zee		 	 	 	 	 	 	 	 	 dde
j        d	z  de
j        d	z  de
j        d	z  ded	z  de
j        d	z  de
j        d	z  ded	z  ded	z  dee
j        z  dee         defd„¦   «         ¦   «         Zˆ xZS )ÚAfmoeForCausalLMzlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr   ÚlogitsÚkeep_full_weightc                 óþ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _        t          j        |j        |j        d¬¦  «        | _        |  	                    ¦   «          d S r‡   )
r+   r,   rN  r7  rV  r   r‹   rD   rq  r_  rš   s     €r:   r,   zAfmoeForCausalLM.__init__`  sh   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý Ñ'Ô'ˆŒ
Ø Ô+ˆŒÝ”y Ô!3°VÔ5FÈUÐSÑSÔSˆŒØ�ŠÑÔÐÐÐr;   Nr   r`  ró   r_   r  ra  Úlabelsr.  Úoutput_router_logitsÚlogits_to_keeprJ   r<   c
                 ór  — |�|n| j         j        } | j        d|||||||dœ|
¤Ž}|j        }t	          |	t
          ¦  «        rt          |	 d¦  «        n|	}|                      |dd…|dd…f         ¦  «        }d}|� | j        ||| j	        fi |
¤Ž}t          |||j        |j        |j        |j        ¬¦  «        S )aÏ  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, AfmoeForCausalLM

        >>> model = AfmoeForCausalLM.from_pretrained("meta-afmoe/Afmoe-2-7b-hf")
        >>> tokenizer = AutoTokenizer.from_pretrained("meta-afmoe/Afmoe-2-7b-hf")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)r`  ró   r_   r  ra  r.  rw  )Úlossrs  r  r   r8  r¦   r0  )r&   rw  r7  rd  rW   r"  Úslicerq  Úloss_functionrV  r   r  r   r8  r¦   )r7   r`  ró   r_   r  ra  rv  r.  rw  rx  rJ   Úoutputsr   Úslice_indicesrs  rz  s                   r:   rd   zAfmoeForCausalLM.forwardg  s  € ðB %9Ð$DÐ Ð È$Ì+ÔJjð 	ð +5¨$¬*ð 	+
ØØ)Ø%Ø+Ø'ØØ!5ð	+
ð 	+
ð ð	+
ð 	+
ˆð  Ô1ˆÝ8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô% f¨f°d´oÐPÐPÈÐPÐPˆDå(ØØØ#Ô3Ø!Ô/ØÔ)Ø!Ô/ð
ñ 
ô 
ð 	
r;   )	NNNNNNNNr   )rf   rg   rh   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr,   r   r   rF   r2  rk   r   r4  r3  r"  r   r   r   rd   rm   rn   s   @r:   rp  rp  X  sg  ø€ € € € € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HàÐ/Ð0€Jðð ð ð ð ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø,0Ø-.ð<
ð <
àÔ# dÑ*ð<
ð œ tÑ+ð<
ð Ô&¨Ñ-ð	<
ð
  ™ð<
ð Ô(¨4Ñ/ð<
ð Ô  4Ñ'ð<
ð ˜$‘;ð<
ð # T™kð<
ð ˜eœlÑ*ð<
ð Ð+Ô,ð<
ð 
#ð<
ð <
ð <
ñ „^ñ Ôð<
ð <
ð <
ð <
ð <
r;   rp  )rp  rN  r6  )r    )rî   )DÚcollections.abcr   rF   r   Ú r   r>  Úactivationsr   Úcache_utilsr   r	   Ú
generationr
   Úintegrationsr   r   r   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   r   Úconfiguration_afmoer!   ÚModuler#   rq   r…   r“   r®   rÍ   rÞ   ræ   rk   r"  rí   rH   r  r  r$  r6  rN  rp  Ú__all__r0  r;   r:   ú<module>r–     sõ  ðð* %Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )Ø hÐ hÐ hÐ hÐ hÐ hÐ hÐ hÐ hÐ hØ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø QÐ QÐ QÐ QÐ QÐ QÐ QÐ QØ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ ,Ð ,Ð ,Ð ,Ð ,Ð ,ð6<ð 6<ð 6<ð 6<ð 6<˜2œ9ñ 6<ô 6<ð 6<ðr Ð˜YÑ'Ô'ðJð Jð Jð Jð J�2”9ñ Jô Jñ (Ô'ðJð(ð ð ð ð ˆrŒyñ ô ð ð ;ð ;ð ;ð ;ð ;˜RœYñ ;ô ;ð ;ð< ð$#ð $#ð $#ð $#ð $#�2”9ñ $#ô $#ñ Ôð$#ðN-ð -ð -ð -ð -˜"œ)ñ -ô -ð -ð>(ð (ð (ð ÐÐ-Ñ.Ô.ðð ð ñ /Ô.ðð2	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð& ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð2 ÐÐ)Ñ*Ô*ðV)ð V)ð V)ð V)ð V)�R”Yñ V)ô V)ñ +Ô*ðV)ðr?ð ?ð ?ð ?ð ?Ð2ñ ?ô ?ð ?ðD*,ð *,ð *,ð *,ð *,˜?ñ *,ô *,ð *,ðZ ðV
ð V
ð V
ð V
ð V
Ð%ñ V
ô V
ñ „ðV
ðr ðL
ð L
ð L
ð L
ð L
Ð+¨_ñ L
ô L
ñ „ðL
ð^ EÐ
DÐ
D€€€r;   