§
    û¸¸jä3  ã                   óv   — d dl mZ d dlZd dlmZ d dlmZ d dlm	Z	  ee
¦  «        Zd„ Z G d„ de¦  «        ZdS )	é    )Ú	getLoggerN)ÚFusionGptAttentionPastBase)Úhelper)Ú	OnnxModelc                 ó.   — t          | |z
  ¦  «        dk    S )Ng�íµ ÷Æ°>)Úabs)ÚvalueÚexpected_values     ú†/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/onnxruntime/transformers/fusion_gpt_attention_megatron.pyÚis_closer      s   € Ýˆu�~Ñ%Ñ&Ô&¨$Ò.Ð.ó    c                   ó>   ‡ — e Zd ZdZdedefˆ fd„Zd„ Zd„ Zd„ Z	ˆ xZ
S )ÚFusionGptAttentionMegatronz^
    Fuse GPT-2 Attention with past state subgraph from Megatron into one Attention node.
    ÚmodelÚ	num_headsc                 óL   •— t          ¦   «                              ||¦  «         d S )N)ÚsuperÚ__init__)Úselfr   r   Ú	__class__s      €r   r   z#FusionGptAttentionMegatron.__init__   s#   ø€ Ý‰Œ×Ò˜ 	Ñ*Ô*Ð*Ð*Ð*r   c                 ó   — | j                              d¦  «        }|                      |¦  «        }	|j        d         }
|j        d         |j        d         k    rdnd}t          j        d||j        d         |j        |         |	|g|
|g|¬¦  «        }d|_        |j         	                    t          j
        d| j        ¦  «        t          j
        dd¦  «        g¦  «         | j        �@|j         	                    t          j
        d	t          | j        ¦  «        ¦  «        g¦  «         |g}| j         	                    |¦  «         |D ]}| j        | j        |j        <   Œ| j                             |¦  «         d
| _        d S )NÚGptAttentionr   é   Ú	Attention)ÚinputsÚoutputsÚnamezcom.microsoftr   ÚunidirectionalÚmask_filter_valueT)r   Úcreate_node_nameÚcast_attention_maskÚoutputÚinputr   Ú	make_nodeÚdomainÚ	attributeÚextendÚmake_attributer   r   ÚfloatÚnodes_to_addÚthis_graph_nameÚnode_name_to_graph_namer   Únodes_to_removeÚappendÚprune_graph)r   Úmatmul_before_splitÚadd_before_splitÚpastÚpresentr#   Úreshape_qkvÚmaskÚattention_node_nameÚ
int32_maskr"   ÚiÚattention_noder*   Únodes                  r   Úfuse_attention_nodez.FusionGptAttentionMegatron.fuse_attention_node   s£  € ð #œj×9Ò9¸.ÑIÔIÐØ×-Ò-¨dÑ3Ô3ˆ
ØÔ# AÔ&ˆØ"Ô(¨Ô+Ð/BÔ/IÈ!Ô/LÒLÐLˆAˆAÐSTˆÝÔ)ØàØ#Ô)¨!Ô,Ø Ô& qÔ)ØØðð ˜WÐ%Ø$ð
ñ 
ô 
ˆð !0ˆÔØÔ ×'Ò'åÔ% k°4´>ÑBÔBÝÔ%Ð&6¸Ñ:Ô:ðñ	
ô 	
ð 	
ð Ô!Ð-ØÔ$×+Ò+­VÔ-BÐCVÕX]Ð^bÔ^tÑXuÔXuÑ-vÔ-vÐ,wÑxÔxÐxà&Ð'ˆØÔ× Ò  Ñ.Ô.Ð.à ð 	Kð 	KˆDØ6:Ô6JˆDÔ(¨¬Ñ3Ð3àÔ×#Ò# KÑ0Ô0Ð0ð  ˆÔÐÐr   c                 óð  — | j                              |g d¢g d¢¦  «        }|€t                               d¦  «         d S |\  }}}}	t	          |¦  «        dk    rB|d         j        dk    r1| j                              |d         ¦  «        \  }
}|dk    r| | _        |j        d         |j	        d         k    rt                               d¦  «         d S | j
                             |dd	¦  «        st                               d
¦  «         d S | j
                             |dd¦  «        st                               d¦  «         d S | j                              |	j        d         ¦  «        st                               d¦  «         d S | j
                             |ddg¦  «        st                               d¦  «         d S | j
                             |ddg¦  «        st                               d¦  «         dS | j
                             |ddg¦  «        st                               d¦  «         dS | j
                             |	ddg¦  «        st                               d¦  «         d S | j
                             |	ddg¦  «        st                               d¦  «         d S | j                              |g d¢g d¢¦  «        }|�|d         |k    rt                               d¦  «         d S | j                              |	g d¢g d¢¦  «        }|�|d         |k    rt                               d¦  «         d S | j                              |	g d¢g d¢¦  «        }|�|d         |k    rt                               d¦  «         d S | j                              |	g d¢g d ¢¦  «        }|€ | j                              |	g d!¢g d ¢¦  «        }|�|d         |k    rt                               d"¦  «         d S |	j        d         S )#N)ÚMulÚSubÚSlicer?   )r   r   r   r   z8fuse_attention: failed to match unidirectional mask pathr   r   r=   i'  zCfuse_attention failed: mul_qk.input[1] != last_slice_mask.output[0]g     ˆÃ@z?fuse_attention failed: mul_mask input 1 is not constant 10000.0g      ð?z;fuse_attention failed: sub_mask input 0 is not constant 1.0z+expect slick_mask input 0 to be graph inputzKfuse_attention failed: last_slice_mask input 1 (starts) is not constant [0]é   zIfuse_attention failed: last_slice_mask input 3 (axes) is not constant [3]Fé   zJfuse_attention failed: last_slice_mask input 4 (steps) is not constant [1]é   zDfuse_attention failed: slice_mask input 3 (axes) is not constant [2]zEfuse_attention failed: slice_mask input 4 (steps) is not constant [1])Ú	UnsqueezeÚGatherÚShapeÚMatMul)rB   r   r   r   éÿÿÿÿz/fuse_attention: failed to match last slice pathz0fuse_attention: failed to match first slice path)rC   r>   rD   rE   rF   )r   r   r   r   r   z3fuse_attention: failed to match last slice sub path)rC   r>   rD   rE   ÚLayerNormalization)r   r   r   r   r   )rC   r>   rD   rE   ÚSkipLayerNormalizationz5fuse_attention: failed to match last slice sub path 1)r   Úmatch_parent_pathÚloggerÚdebugÚlenÚop_typeÚget_constant_inputr   r#   r"   ÚutilsÚcheck_node_input_valueÚfind_graph_inputÚinfo)r   Úsub_qkÚmul_qkÚ	matmul_qkÚlayernorm_before_attentionÚ
mask_nodesÚmul_maskÚsub_maskÚlast_slice_maskÚ
slice_maskÚ_Úmul_valÚlast_slice_pathÚfirst_slice_pathÚfirst_slice_subÚfirst_slice_sub_1s                   r   Ú
match_maskz%FusionGptAttentionMegatron.match_maskJ   s0  € Ø”Z×1Ò1°&Ð:ZÐ:ZÐ:ZÐ\hÐ\hÐ\hÑiÔiˆ
ØÐÝ�LŠLÐSÑTÔTÐTØ�4Ø<FÑ9ˆ�8˜_¨jåˆz‰?Œ?˜QÒÐ :¨a¤=Ô#8¸EÒ#AÐ#AØœ×6Ò6°zÀ!´}ÑEÔE‰JˆAˆwØ˜%ÒÐØ*1¨�Ô&àŒ<˜Œ?˜oÔ4°QÔ7Ò7Ð7Ý�LŠLÐ^Ñ_Ô_Ð_Ø�4àŒz×0Ò0°¸1¸gÑFÔFð 	Ý�LŠLÐZÑ[Ô[Ð[Ø�4àŒz×0Ò0°¸1¸cÑBÔBð 	Ý�LŠLÐVÑWÔWÐWØ�4àŒz×*Ò*¨:Ô+;¸AÔ+>Ñ?Ô?ð 	Ý�KŠKÐEÑFÔFÐFØ�4àŒz×0Ò0°À!ÀaÀSÑIÔIð 	Ý�LŠLÐfÑgÔgÐgØ�4àŒz×0Ò0°À!ÀaÀSÑIÔIð 	Ý�LŠLÐdÑeÔeÐeØ�5àŒz×0Ò0°À!ÀaÀSÑIÔIð 	Ý�LŠLÐeÑfÔfÐfØ�5àŒz×0Ò0°¸QÀÀÑDÔDð 	Ý�LŠLÐ_Ñ`Ô`Ð`Ø�4àŒz×0Ò0°¸QÀÀÑDÔDð 	Ý�LŠLÐ`ÑaÔaÐaØ�4àœ*×6Ò6ØÐGÐGÐGÈÈÈñ
ô 
ˆð Ð" o°bÔ&9¸YÒ&FÐ&FÝ�LŠLÐJÑKÔKÐKØ�4àœ:×7Ò7ØÐBÐBÐBÀLÀLÀLñ
ô 
Ðð Ð#Ð'7¸Ô';¸yÒ'HÐ'HÝ�LŠLÐKÑLÔLÐLØ�4àœ*×6Ò6ØØ=Ð=Ð=ØˆOˆOñ
ô 
ˆð
 Ð" o°bÔ&9¸YÒ&FÐ&FÝ�LŠLÐNÑOÔOÐOØ�4à œJ×8Ò8ØØIÐIÐIØˆOˆOñ
ô 
Ðð Ð$Ø $¤
× <Ò <ØØQÐQÐQØ��ñ!ô !Ðð Ð$Ð(9¸"Ô(=ÐA[Ò([Ð([Ý�LŠLÐPÑQÔQÐQØ�4àÔ Ô"Ð"r   c           	      óü
  — d }d }|j         dk    }d }|s#| j                             |g d¢g d¢|¬¦  «        }n"| j                             |g d¢g d¢|¬¦  «        }|€d S d }|s|\  }	}
}}}}|	j        d         }n|\  }
}}}}|j        d         }| j                             |g d¢g d	¢¦  «        }|€ | j                             |g d
¢g d	¢¦  «        }|€t                               d¦  «         d S |\  }}}}}}}|j         dk    r-||j        d         k    rt                               d¦  «         d S |j         dk    r-||j        d         k    rt                               d¦  «         d S | j                             |g d¢g d¢¦  «        }|€t                               d¦  «         d S |\  }}}}| j                             |d¦  «        dk    rt                               d¦  «         d S |                      ||||¦  «        }| j                             |g d¢g d¢¦  «        }|€t                               d¦  «         d S |\  }}} }!||!k    rt                               d¦  «         d S | j                             |g d¢g d¢¦  «        }"|"€t                               d¦  «         d S |"\  }#}$}%}&}'}(||(k    rt                               d¦  «         d S | j         	                    |'¦  «        \  })}*t          |*t          j        ¦  «        rIt          |*j        ¦  «        dgk    r0|*d         dk    r$|*d         dk    r|*d         dk    r|*d         dk    st                               d¦  «         d S |*d         }+|+| j        k    r,t                               d|+› d | j        › �¦  «         |+| _        |*d         },| j         	                    |#¦  «        \  })}*t#          t          j        t          j        |,¦  «        ¦  «        ¦  «        }-t'          |*|-¦  «        s"t                               d!|*› d"|-› �¦  «         d S | j         	                    |¦  «        \  })}*t'          |*|-¦  «        s"t                               d#|*› d"|-› �¦  «         d S |                      |%||¦  «        }|€t                               d$¦  «         d S | j                             |¦  «        st                               d%¦  «         |                      ||¦  «        }|€t                               d&¦  «         d S | j                             |¦  «        st                               d'¦  «         d S |                      |||||j        d         ||¦  «         d S )(NrI   )ÚAddre   rF   ÚReshapeÚ	TransposerF   )r   r   Nr   r   r   )Úoutput_name_to_node)re   rF   rf   rg   rF   )r   Nr   r   r   r   )ÚConcatrg   rf   ÚSplitre   rF   rH   )r   r   r   r   r   Nr   )ri   rg   rf   rj   re   rF   rI   z&fuse_attention: failed to match v pathrH   zAfuse_attention: skip_input != layernorm_before_attention.input[0]r@   )ÚSoftmaxr>   r=   rF   )r   r   r   r   z'fuse_attention: failed to match qk pathÚaxisz+fuse_attention failed: softmax_qk axis != 3)ÚDivrg   rf   rj   z&fuse_attention: failed to match q pathz-fuse_attention: skip since split_v != split_q)rm   rg   ri   rg   rf   rj   )r   r   r   r   r   r   z&fuse_attention: failed to match k pathz-fuse_attention: skip since split_v != split_krA   r   rB   z:fuse_attention: reshape constant input is not [0, 0, N, H]zDetected num_heads=z. Ignore user specified value zfuse_attention: div_k value=z
 expected=zfuse_attention: div_q value=z!fuse_attention: match past failedz(fuse_attention: past is not graph input.z$fuse_attention: match present failedz1fuse_attention: expect present to be graph output)rN   r   rJ   r#   rK   rL   r"   Úget_node_attributerc   rO   Ú
isinstanceÚnpÚndarrayÚlistÚshaper   rS   r)   Úsqrtr   Úmatch_past_pattern_2rR   Úmatch_presentÚfind_graph_outputr;   ).r   Únormalize_nodeÚinput_name_to_nodesrh   r2   r3   Úis_normalize_node_skiplayernormÚ	qkv_nodesÚ
skip_inputÚadd_skipÚadd_after_attentionÚmatmul_after_attentionr4   Útranspose_qkvÚ
matmul_qkvÚv_nodesÚconcat_vÚtranspose_vÚ	reshape_vÚsplit_vr1   r0   rW   Úqk_nodesÚ
softmax_qkrT   rU   rV   Úattention_maskÚq_nodesÚdiv_qÚtranspose_qÚ	reshape_qÚsplit_qÚk_nodesÚdiv_kr]   Úconcat_kÚtranspose_kÚ	reshape_kÚsplit_kr8   r	   r   Úhidden_size_per_headr
   s.                                                 r   ÚfusezFusionGptAttentionMegatron.fuse¤   sk  € ØˆØˆà*8Ô*@ÐD\Ò*\Ð'Øˆ	à.ð 	Øœ
×4Ò4ØØJÐJÐJØ%Ð%Ð%Ø$7ð	 5ñ ô ˆIˆIð œ
×4Ò4ØØCÐCÐCØ"Ð"Ð"Ø$7ð	 5ñ ô ˆIð ÐØˆFàˆ
Ø.ð 	1ð ñØØ#Ø&ØØØð "œ¨Ô*ˆJˆJð ñØ#Ø&ØØØð (Ô-¨aÔ0ˆJà”*×.Ò.Øðð ð ð %Ð$Ð$ñ
ô 
ˆð ˆ?Ø”j×2Ò2Øðð ð ð )Ð(Ð(ñô ˆGð ˆ?Ý�LŠLÐAÑBÔBÐBØˆFð ñ	
ØØØØØØØ&ð 'Ô.Ð2FÒFÐFØÐ8Ô>¸qÔAÒAÐAå�LŠLÐ\Ñ]Ô]Ð]ØˆFð 'Ô.Ð2JÒJÐJØÐ8Ô?ÀÔBÒBÐBå�LŠLÐ\Ñ]Ô]Ð]ØˆFà”:×/Ò/°
Ð<_Ð<_Ð<_ÐamÐamÐamÑnÔnˆØÐÝ�LŠLÐBÑCÔCÐCØ�4Ø2:Ñ/ˆ�V˜V YØŒ:×(Ò(¨°VÑ<Ô<ÀÒAÐAÝ�LŠLÐFÑGÔGÐGØ�4àŸš¨°¸ÐD^Ñ_Ô_ˆà”*×.Ò.¨yÐ:bÐ:bÐ:bÐdpÐdpÐdpÑqÔqˆØˆ?Ý�LŠLÐAÑBÔBÐBØˆFØ3:Ñ0ˆ�˜Y¨Ø�gÒÐÝ�LŠLÐHÑIÔIÐIØˆFà”*×.Ò.ØØKÐKÐKØÐÐñ
ô 
ˆð
 ˆ?Ý�LŠLÐAÑBÔBÐBØˆFØ@GÑ=ˆ��8˜[¨)°WØ�gÒÐÝ�LŠLÐHÑIÔIÐIØˆFà”:×0Ò0°Ñ;Ô;‰ˆˆ5å�u�bœjÑ)Ô)ð		å�U”[Ñ!Ô! a SÒ(Ð(Ø�a”˜A’�Ø�a”˜A’�Ø�a”˜1’�Ø�a”˜1’�å�LŠLÐUÑVÔVÐVØˆFà˜!”Hˆ	Ø˜œÒ&Ð&Ý�KŠKÐg¨iÐgÐgÐW[ÔWeÐgÐgÑhÔhÐhØ&ˆDŒNà$ QœxÐØ”:×0Ò0°Ñ7Ô7‰ˆˆ5Ý�rœw¥r¤wÐ/CÑ'DÔ'DÑEÔEÑFÔFˆÝ˜˜~Ñ.Ô.ð 	Ý�LŠLÐY¸ÐYÐYÈÐYÐYÑZÔZÐZØˆFà”:×0Ò0°Ñ7Ô7‰ˆˆ5Ý˜˜~Ñ.Ô.ð 	Ý�LŠLÐY¸ÐYÐYÈÐYÐYÑZÔZÐZØˆFð ×(Ò(¨°8Ð=PÑQÔQˆØˆ<Ý�LŠLÐ<Ñ=Ô=Ð=ØˆFØŒz×*Ò*¨4Ñ0Ô0ð 	EÝ�LŠLÐCÑDÔDÐDð ×$Ò$ XÐ/BÑCÔCˆØˆ?Ý�LŠLÐ?Ñ@Ô@Ð@ØˆFØŒz×+Ò+¨GÑ4Ô4ð 	Ý�KŠKÐKÑLÔLÐLØˆFà× Ò ØØØØØ&Ô-¨aÔ0ØØñ	
ô 	
ð 	
ð 	
ð 	
r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Úintr   r;   rc   r–   Ú__classcell__)r   s   @r   r   r      sŽ   ø€ € € € € ðð ð+˜ið +°Cð +ð +ð +ð +ð +ð +ð- ð - ð - ð^X#ð X#ð X#ðt
ð 
ð 
ð 
ð 
ð 
ð 
r   r   )Úloggingr   Únumpyrp   Úfusion_gpt_attentionr   Úonnxr   Ú
onnx_modelr   r—   rK   r   r   © r   r   ú<module>r£      s¸   ðð
 Ð Ð Ð Ð Ð à Ð Ð Ð Ø ;Ð ;Ð ;Ð ;Ð ;Ð ;Ø Ð Ð Ð Ð Ð Ø  Ð  Ð  Ð  Ð  Ð  à	ˆ�8Ñ	Ô	€ð/ð /ð /ðP
ð P
ð P
ð P
ð P
Ð!;ñ P
ô P
ð P
ð P
ð P
r   