§
    û¸¸jl�  ã                   ó  — d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dl	Z	d dl
Z
d dlZd dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZmZmZmZ  e j        e ¦  «        Z!g d¢Z"ej#        dej$        dej%        diZ& G d„ de¦  «        Z' G d„ de¦  «        Z( G d„ de¦  «        Z) G d„ de¦  «        Z* G d„ de¦  «        Z+e*ddfe+ddfe)ddfdœZ, G d„ d¦  «        Z- G d „ d!¦  «        Z.dS )"é    N)ÚPath)Ú	Precision)Úfloat_to_float16_max_diff)ÚFusionOptions)ÚIOBindingHelper)Ú	OnnxModel)Úoptimize_model)Útorch_onnx_export)Ú
GPT2ConfigÚGPT2LMHeadModelÚ	GPT2ModelÚTFGPT2Model)Ú
distilgpt2Úgpt2zgpt2-mediumz
gpt2-largezgpt2-xlçü©ñÒMb@?gš™™™™™É?g      @c                   ó,   ‡ — e Zd ZdZˆ fd„Zˆ fd„Zˆ xZS )ÚGPT2ModelNoPastStateú2Here we wrap a class to disable past state output.c                 óJ   •— t          ¦   «                              |¦  «         d S ©N©ÚsuperÚ__init__©ÚselfÚconfigÚ	__class__s     €ú€/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/onnxruntime/transformers/models/gpt2/gpt2_helper.pyr   zGPT2ModelNoPastState.__init__*   ó!   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ð Ð ó    c                 óL   •— t          ¦   «                              |dd¬¦  «        S )NF)Ú	use_cacheÚreturn_dict)r   Úforward©r   Ú	input_idsr   s     €r   r$   zGPT2ModelNoPastState.forward-   s   ø€ Ý‰wŒw�Š˜y°EÀuˆÑMÔMÐMr    ©Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r$   Ú__classcell__©r   s   @r   r   r   '   s`   ø€ € € € € Ø<Ð<ð!ð !ð !ð !ð !ðNð Nð Nð Nð Nð Nð Nð Nð Nr    r   c                   ó,   ‡ — e Zd ZdZˆ fd„Zˆ fd„Zˆ xZS )ÚTFGPT2ModelNoPastStater   c                 óX   •— d|_         t          ¦   «                              |¦  «         d S )NF)r"   r   r   r   s     €r   r   zTFGPT2ModelNoPastState.__init__4   s)   ø€ Ø ˆÔÝ‰Œ×Ò˜Ñ Ô Ð Ð Ð r    c                 óJ   •— t          ¦   «                              |d¬¦  «        S )NF)r"   )r   Úcallr%   s     €r   r$   zTFGPT2ModelNoPastState.forward8   s   ø€ Ý‰wŒw�|Š|˜I°ˆ|Ñ7Ô7Ð7r    r'   r-   s   @r   r/   r/   1   sW   ø€ € € € € Ø<Ð<ð!ð !ð !ð !ð !ð8ð 8ð 8ð 8ð 8ð 8ð 8ð 8ð 8r    r/   c                   óB   ‡ — e Zd ZdZˆ fd„Zed„ ¦   «         Zˆ fd„Zˆ xZS )ÚMyGPT2ModelzMHere we wrap a class for Onnx model conversion for GPT2Model with past state.c                 óJ   •— t          ¦   «                              |¦  «         d S r   r   r   s     €r   r   zMyGPT2Model.__init__?   r   r    c           	      ó
  — t          | d         d         t          t          f¦  «        rÚt          | d         ¦  «        |k    rt          | d         d         ¦  «        dk    sJ ‚g }t	          |¦  «        D ]w}|                     t          j        | d         |         d                              d¦  «        | d         |         d                              d¦  «        fd¬¦  «        ¦  «         Œx| d         t          |¦  «        fS | S )Né   r   é   )Údim)	Ú
isinstanceÚtupleÚlistÚlenÚrangeÚappendÚtorchÚcatÚ	unsqueeze)ÚresultÚ	num_layerÚpresentÚis       r   Úpost_processzMyGPT2Model.post_processB   sþ   € å�f˜Q”i ”l¥U­D MÑ2Ô2ð 	/Ý�v˜a”y‘>”> YÒ.Ð.µ3°v¸a´yÀ´|Ñ3DÔ3DÈÒ3IÐ3IÐ3IÐIàˆGÝ˜9Ñ%Ô%ð ð �ð —’Ý”IØ œ 1œ aœ×2Ò2°1Ñ5Ô5°v¸a´yÀ´|ÀA´×7PÒ7PÐQRÑ7SÔ7SÐTØðñ ô ñô ð ð ð ˜1”I�u W™~œ~Ð.Ð.àˆr    c                 óš   •— t          ¦   «                              ||||d¬¦  «        }t                               || j        j        ¦  «        S ©NF)Úposition_idsÚattention_maskÚpast_key_valuesr#   ©r   r$   r4   rG   r   Ún_layer©r   r&   rJ   rK   ÚpastrC   r   s         €r   r$   zMyGPT2Model.forwardU   sK   ø€ Ý‘”—’ØØ%Ø)Ø Øð !ñ 
ô 
ˆõ ×'Ò'¨°´Ô0CÑDÔDÐDr    )	r(   r)   r*   r+   r   ÚstaticmethodrG   r$   r,   r-   s   @r   r4   r4   <   s{   ø€ € € € € ØWÐWð!ð !ð !ð !ð !ð ðð ñ „\ðð$Eð Eð Eð Eð Eð Eð Eð Eð Er    r4   c                   ó,   ‡ — e Zd ZdZˆ fd„Zˆ fd„Zˆ xZS )ÚMyGPT2LMHeadModelzSHere we wrap a class for Onnx model conversion for GPT2LMHeadModel with past state.c                 óJ   •— t          ¦   «                              |¦  «         d S r   r   r   s     €r   r   zMyGPT2LMHeadModel.__init__c   r   r    c                 óš   •— t          ¦   «                              ||||d¬¦  «        }t                               || j        j        ¦  «        S rI   rM   rO   s         €r   r$   zMyGPT2LMHeadModel.forwardf   sK   ø€ Ý‘”—’ØØ%Ø)Ø Øð !ñ 
ô 
ˆõ ×'Ò'¨°´Ô0CÑDÔDÐDr    r'   r-   s   @r   rS   rS   `   s`   ø€ € € € € Ø]Ð]ð!ð !ð !ð !ð !ð	Eð 	Eð 	Eð 	Eð 	Eð 	Eð 	Eð 	Eð 	Er    rS   c                   ó,   ‡ — e Zd ZdZˆ fd„Zˆ fd„Zˆ xZS )ÚMyGPT2LMHeadModel_NoPaddinga  Here we wrap a class for Onnx model conversion for GPT2LMHeadModel with past state and no padding.
    When you always use batch_size=1 in inference, there is no padding in inputs. In such case, position_ids
    and attention_mask need no be in inputs.
    c                 óJ   •— t          ¦   «                              |¦  «         d S r   r   r   s     €r   r   z$MyGPT2LMHeadModel_NoPadding.__init__x   r   r    c                 ó–   •— t          ¦   «                              ||d¬¦  «        }t                               || j        j        ¦  «        S )NF)rL   r#   rM   )r   r&   rP   rC   r   s       €r   r$   z#MyGPT2LMHeadModel_NoPadding.forward{   s9   ø€ Ý‘”—’ ¸DÈe�ÑTÔTˆå×'Ò'¨°´Ô0CÑDÔDÐDr    r'   r-   s   @r   rW   rW   r   sd   ø€ € € € € ðð ð
!ð !ð !ð !ð !ðEð Eð Eð Eð Eð Eð Eð Eð Er    rW   ÚlogitsTFÚ
last_state)r   ÚGPT2LMHeadModel_NoPaddingr   c                   ó2   — e Zd Zd„ Zdefd„Zdefd„Zd„ ZdS )Ú
Gpt2Inputsc                 ó>   — || _         || _        || _        || _        d S r   )r&   rJ   rK   rP   )r   r&   rJ   rK   rP   s        r   r   zGpt2Inputs.__init__Š   s$   € Ø+4ˆŒØ.:ˆÔØWeˆÔØFJˆŒ	ˆ	ˆ	r    Úreturnc                 ó„   — d„ | j         | j        | j        fD ¦   «         }| j        r|                     | j        ¦  «         |S )Nc                 ó   — g | ]}|®|‘ŒS r   © ©Ú.0Úvs     r   ú
<listcomp>z&Gpt2Inputs.to_list.<locals>.<listcomp>‘   s   € ÐkÐkÐk˜AÐ]^Ð]j�aÐ]jÐ]jÐ]jr    )r&   rJ   rK   rP   Úextend)r   Ú
input_lists     r   Úto_listzGpt2Inputs.to_list�   sJ   € ØkÐk $¤.°$Ô2CÀTÔEXÐ!YÐkÑkÔkˆ
ØŒ9ð 	)Ø×Ò˜dœiÑ(Ô(Ð(àÐr    c                 ód   — t          d„ | j        | j        | j        | j        fD ¦   «         ¦  «        S )Nc              3   ó   K  — | ]}|®|V — Œ	d S r   rc   rd   s     r   ú	<genexpr>z&Gpt2Inputs.to_tuple.<locals>.<genexpr>˜   s(   è è € ÐuÐu˜1ÐghÐgt�QÐgtÐgtÐgtÐgtÐuÐur    )r;   r&   rJ   rK   rP   )r   s    r   Úto_tuplezGpt2Inputs.to_tuple—   s4   € ÝÐuÐu ¤°Ô1BÀDÔDWÐY]ÔYbÐ cÐuÑuÔuÑuÔuÐur    c                 óú   — d }| j         �F| j         j        t          j        k    r%| j                              t          j        ¬¦  «        n| j         }d„ | j        D ¦   «         }t          | j        | j	        ||¦  «        S )N©Údtypec                 óN   — g | ]"}|                      t          j        ¬ ¦  «        ‘Œ#S )rp   )Útor@   Úfloat32©re   Úps     r   rg   z&Gpt2Inputs.to_fp32.<locals>.<listcomp>¤   s(   € Ð=Ð=Ð=¨a�—’�5œ=�Ñ)Ô)Ð=Ð=Ð=r    )
rK   rq   r@   Úfloat16rs   rt   rP   r^   r&   rJ   )r   rK   rP   s      r   Úto_fp32zGpt2Inputs.to_fp32š   s~   € àˆØÔÐ*ð Ô'Ô-µ´Ò>Ð>ð Ô#×&Ò&­U¬]Ð&Ñ;Ô;Ð;àÔ(ð ð >Ð=°4´9Ð=Ñ=Ô=ˆÝ˜$œ.¨$Ô*;¸^ÈTÑRÔRÐRr    N)	r(   r)   r*   r   r<   rj   r;   rn   rx   rc   r    r   r^   r^   ‰   su   € € € € € ðKð Kð Kð˜ð ð ð ð ðv˜%ð vð vð vð vðSð Sð Sð Sð Sr    r^   c            "       óL  — e Zd ZdZedddej        ej        ej        dfdededededed	ed
edej        de	de	de	dej
        dej
        dej
        de	def d„¦   «         Ze	 dCdedededededeeee         f         fd„¦   «         Zed„ ¦   «         ZedDd„¦   «         ZedDd„¦   «         ZedEd„¦   «         ZedFd„¦   «         Zeddddej        ej        ej        fd ed!e	d"e	de	de	dej
        dej
        dej
        fd#„¦   «         Ze	 	 	 dGd%„¦   «         Zeg d&¢fd'ed(ee         fd)„¦   «         ZedHd*ed+efd,„¦   «         ZedHd*ed+efd-„¦   «         Zed.„ ¦   «         ZedId/„¦   «         Ze	 	 	 dJd*ed0eeej        f         d1eeee         f         d+ed2e	d3e	fd4„¦   «         Z ed5„ ¦   «         Z!ed6„ ¦   «         Z"edd7d7d8d9ddddej        ej        ej        d$ddfd:„¦   «         Z#edd;ddddej        ej        ej        d<d9d=fd>„¦   «         Z$edKd?„¦   «         Z%edddg d@¢fdefdA„¦   «         Z&dBS )LÚ
Gpt2HelperzEA helper class for Gpt2 model conversion, inference and verification.FTÚ
batch_sizeÚpast_sequence_lengthÚsequence_lengthÚnum_attention_headsÚhidden_sizerD   Ú
vocab_sizeÚdevicerw   Úhas_position_idsÚhas_attention_maskÚinput_ids_dtypeÚposition_ids_dtypeÚattention_mask_dtypeÚleft_side_paddingr`   c                 óª  ‡‡‡— |rt           j        nt           j        Šd| ||t          ||z  ¦  «        gŠˆˆˆfd„t	          |¦  «        D ¦   «         }t          j        d|dz
  | |f|‰¬¦  «        }d}|
rf||z   }t          j        | |g|‰¬¦  «        }|dk    rBt	          | ¦  «        D ]2}t          j        d|dz
  ¦  «        }|r
d||d|…f<   Œ&d||||z
  d…f<   Œ3d}|	re|                     ¦   «          	                    d¦  «        dz
  }| 
                    |dk     d¦  «         |dd…|d…f                              |¦  «        }t          ||||¦  «        S )	z˜Create random inputs for GPT2 model.
        Returns torch tensors of input_ids, position_ids, attention_mask and a list of past state tensors.
        r8   c                 óJ   •— g | ]}t          j        ‰‰‰¬ ¦  «        dz  dz
  ‘Œ S )©rq   r�   g       @ç      ð?)r@   Úrand)re   Ú_r�   Ú
float_typeÚ
past_shapes     €€€r   rg   z/Gpt2Helper.get_dummy_inputs.<locals>.<listcomp>É   s6   ø€ ÐpÐpÐpÐZ[•”˜J¨jÀÐHÑHÔHÈ3ÑNÐQTÑTÐpÐpÐpr    r   r7   )ÚlowÚhighÚsizerq   r�   NrŠ   éÿÿÿÿ)r@   rw   rt   Úintr>   ÚrandintÚonesÚrandomÚlongÚcumsumÚmasked_fill_rs   r^   )r{   r|   r}   r~   r   rD   r€   r�   rw   r‚   rƒ   r„   r…   r†   r‡   rP   r&   rK   Útotal_sequence_lengthrF   Úpadding_lengthrJ   rŽ   r�   s          `              @@r   Úget_dummy_inputszGpt2Helper.get_dummy_inputs«   sÝ  øøø€ ð* '.Ð@•U”]�]µ5´=ˆ
àØØØ Ý�Ð1Ñ1Ñ2Ô2ð
ˆ
ð qÐpÐpÐpÐpÐpÕ_dÐenÑ_oÔ_oÐpÑpÔpˆÝ”MØØ˜a‘Ø˜oÐ.Ø!Øð
ñ 
ô 
ˆ	ð ˆØð 	XØ$8¸?Ñ$JÐ!Ý"œZØÐ2Ð3Ø*Øðñ ô ˆNð %¨Ò)Ð)Ý˜zÑ*Ô*ð Xð X�AÝ%+¤^°AÐ7LÈqÑ7PÑ%QÔ%Q�NØ(ð XØ=>˜ q¨/¨>¨/Ð'9Ñ:Ð:àVW˜ qÐ*?À.Ñ*PÐ*RÐ*RÐ'RÑSÐSð ˆØð 	YØ)×.Ò.Ñ0Ô0×7Ò7¸Ñ;Ô;¸aÑ?ˆLØ×%Ò% l°QÒ&6¸Ñ:Ô:Ð:Ø'¨¨¨Ð+?Ð+@Ð+@Ð(@ÔA×DÒDÐEWÑXÔXˆLå˜) \°>À4ÑHÔHÐHr    r   r   Úmodel_classc                 ó  — |j         }|j        }|j        }|j        }t          |         d         }	| ||	dk    r|n|g}
d| |||z   t          ||z  ¦  «        g}|	|
i}t          |¦  «        D ]}||dt          |¦  «        z   <   Œ|S )zAReturns a dictionary with output name as key, and shape as value.r7   rZ   r8   Úpresent_)r~   r   Únum_hidden_layersr€   ÚMODEL_CLASSESr”   r>   Ústr)r{   r|   r}   r   rž   r~   r   rD   r€   Úoutput_nameÚlast_state_shapeÚpresent_state_shapeÚoutput_shapesrF   s                 r   Úget_output_shapeszGpt2Helper.get_output_shapesì   sÅ   € ð %Ô8ÐØÔ(ˆØÔ,ˆ	ØÔ&ˆ
å# KÔ0°Ô3ˆð ØØ%¨Ò1Ð1ˆJˆJ°{ð
Ðð ØØØ  ?Ñ2Ý�Ð1Ñ1Ñ2Ô2ð
Ðð %Ð&6Ð7ˆÝ�yÑ!Ô!ð 	Eð 	EˆAØ1DˆM˜*¥s¨1¡v¤vÑ-Ñ.Ð.àÐr    c                 ó  — |D ]|}|| v sJ ‚| |         }t          j        ||         ¦  «        |                     ¦   «         k    r<t          j        t          j        ||         ¦  «        |j        |j        ¬¦  «        | |<   Œ}d S )NrŠ   )ÚnumpyÚprodÚnelementr@   Úemptyrq   r�   )Úoutput_buffersr§   ÚkeyÚbuffers       r   Úauto_increase_buffer_sizez$Gpt2Helper.auto_increase_buffer_size  s’   € à ð 	ð 	ˆCØ˜.Ð(Ð(Ð(Ð(Ø# CÔ(ˆFÝŒz˜-¨Ô,Ñ-Ô-°·²Ñ0AÔ0AÒAÐAÝ&+¤kÝ”J˜}¨SÔ1Ñ2Ô2Ø œ,Ø!œ=ð'ñ 'ô '�˜sÑ#øð		ð 	r    c                 óÊ   — |rt           j        nt           j        }i }|                      ¦   «         D ]1\  }}t          j        t          j        |¦  «        ||¬¦  «        ||<   Œ2|S )zpReturns a dictionary of output name as key, and 1D tensor as value. The tensor has enough space for given shape.rŠ   )r@   rw   rt   Úitemsr­   rª   r«   )r§   r�   Ú
is_float16Ú	data_typer®   ÚnameÚshapes          r   Úget_output_bufferszGpt2Helper.get_output_buffers  sk   € ð &0ÐB•E”M�MµU´]ˆ	àˆØ(×.Ò.Ñ0Ô0ð 	bð 	b‰KˆD�%Ý#(¤;­u¬z¸%Ñ/@Ô/@È	ÐZ`Ð#aÑ#aÔ#aˆN˜4Ñ Ð ØÐr    c                 ó  — | d                               ¦   «                              ¦   «         }t          j        ||d         z
  ¦  «        }|r,t          j        |t          j        |¦  «        dz   z  ¦  «        S t          j        |¦  «        S )zGReturns the maximum difference between PyTorch and OnnxRuntime outputs.r   ç�íµ ÷Æ°>)Úcpurª   ÚabsÚamax)Útorch_outputsÚort_outputsÚrelativeÚexpected_outputsÚdiffs        r   Údiff_outputszGpt2Helper.diff_outputs%  s}   € ð )¨Ô+×/Ò/Ñ1Ô1×7Ò7Ñ9Ô9ÐÝŒyÐ)¨K¸¬NÑ:Ñ;Ô;ˆØð 	$Ý”:˜d¥e¤iÐ0@Ñ&AÔ&AÀDÑ&HÑIÑJÔJÐJå”:˜dÑ#Ô#Ð#r    çü©ñÒMbP?c           	      ó‚  — t          j        |d         | d                              ¦   «                               ¦   «         ||¬¦  «        }t                               d|› �¦  «         |}t          |¦  «        dz
  }t          |¦  «        D ]z}t          j        |d|z            | d         |                              ¦   «                               ¦   «         ||¬¦  «        }t                               d|› d|› d|› �¦  «         |o|}Œ{|s9t                               | |¦  «        }	t           	                    d|	d	›�¦  «         |S )
zËReturns True if torch and ORT outputs are close for given thresholds, and False otherwise.
        Note: need kwargs since Gpt2BeamSearchHelper.compare_outputs has an extra parameter model_class
        r   )ÚrtolÚatolz9PyTorch and OnnxRuntime output 0 (last_state) are close: r7   zPyTorch and OnnxRuntime layer z state (present_z) are close:z@PyTorch and OnnxRuntime results are not all close: max_abs_diff=ú.5f)
rª   Úallcloser»   ÚloggerÚdebugr=   r>   rz   rÃ   Úinfo)
r¾   r¿   rÆ   rÇ   ÚkwargsÚis_closeÚis_all_closeÚ
num_layersÚlayerÚmax_abs_diffs
             r   Úcompare_outputszGpt2Helper.compare_outputs/  sT  € õ
 ”> +¨a¤.°-ÀÔ2B×2FÒ2FÑ2HÔ2H×2NÒ2NÑ2PÔ2PÐW[ÐbfÐgÑgÔgˆÝ�ŠÐ[ÐQYÐ[Ð[Ñ\Ô\Ð\àˆÝ˜Ñ%Ô%¨Ñ)ˆ
å˜:Ñ&Ô&ð 	5ð 	5ˆEÝ”~Ø˜A ™IÔ&Ø˜aÔ  Ô'×+Ò+Ñ-Ô-×3Ò3Ñ5Ô5ØØð	ñ ô ˆHõ �LŠLÐn¸%ÐnÐnÐQVÐnÐnÐdlÐnÐnÑoÔoÐoØ'Ð4¨HˆLˆLàð 	oÝ%×2Ò2°=À+ÑNÔNˆLÝ�KŠKÐmÐ[gÐmÐmÐmÑnÔnÐnàÐr    rº   c                 ó   — d}d}g }g }t          t          |¦  «        ¦  «        D �]³}||         }|dk    r| d         n| d         |dz
                                ¦   «                              ¦   «         }	t          j        ||	|d¬¦  «        }
|                     t          j        t          j        |	|z
  ¦  «        ¦  «        ¦  «         |o|
}t          j        |	¦  «         	                    ¦   «         rt                               d|› d�¦  «         t          j        |	¦  «         	                    ¦   «         rt                               d|› d�¦  «         t          j        |¦  «         	                    ¦   «         rt                               d	|› d�¦  «         t          j        |¦  «         	                    ¦   «         rt                               d	|› d�¦  «         t          j        ||	z
  ¦  «        }t          j        |                     ¦   «         |j        ¦  «        }|                     d
||         d›d|› d||         d›dt#          |	|         ¦  «        d›�¦  «         |dk    rqt          j        t          j        |d¬¦  «        |j        ¦  «        }t          j        t          j        |	d¬¦  «        |	j        ¦  «        }t          j        ||¦  «        }�Œµ|                     t)          |¦  «        ¦  «        }|t)          |¦  «        |||fS )aë  Compare outputs from PyTorch and OnnxRuntime

        Args:
            torch_outputs (Tuple[Torch.Tensor]): PyTorch model output
            ort_outputs (List[numpy.ndarray]): OnnxRuntime output
            atol (float, optional): Absolute tollerance. Defaults to 1e-06.

        Returns:
            is_all_close(bool): whether all elements are close.
            max_abs_diff(float): maximum absolute difference.
            messages(str): a list of debug message for each output
        TFr   r7   )rÇ   rÆ   zPyTorch output z has nanz has infzORT output zdiff=z.9fz index=z ort=z torch=N)Úaxis)r>   r=   r»   rª   rÉ   r?   r½   r¼   ÚisnanÚanyrÊ   rË   ÚisinfÚfabsÚunravel_indexÚargmaxr·   ÚfloatÚarray_equalÚindexÚmax)r¾   r¿   rÇ   rÏ   Úis_top1_matchedÚ	max_diffsÚmessagesrF   Ú
ort_outputÚtorch_outputrÎ   rÂ   ÚidxÚort_max_indexÚtorch_max_indexÚmax_diff_output_indexs                   r   Úcompare_outputs_v2zGpt2Helper.compare_outputs_v2J  së  € ð ˆØˆØˆ	ØˆÝ•s˜;Ñ'Ô'Ñ(Ô(ð 	Tñ 	TˆAØ$ QœˆJØ01°Q²°˜M¨!Ô,Ð,¸MÈ!Ô<LÈQÐQRÉUÔ<S×XÒXÑZÔZ×`Ò`ÑbÔbˆLÝ”~ j°,ÀTÐPQÐRÑRÔRˆHØ×Ò�UœZ­¬	°,ÀÑ2KÑ(LÔ(LÑMÔMÑNÔNÐNØ'Ð4¨HˆLåŒ{˜<Ñ(Ô(×,Ò,Ñ.Ô.ð <Ý—’Ð:¨qÐ:Ð:Ð:Ñ;Ô;Ð;ÝŒ{˜<Ñ(Ô(×,Ò,Ñ.Ô.ð <Ý—’Ð:¨qÐ:Ð:Ð:Ñ;Ô;Ð;ÝŒ{˜:Ñ&Ô&×*Ò*Ñ,Ô,ð 8Ý—’Ð6¨1Ð6Ð6Ð6Ñ7Ô7Ð7ÝŒ{˜:Ñ&Ô&×*Ò*Ñ,Ô,ð 8Ý—’Ð6¨1Ð6Ð6Ð6Ñ7Ô7Ð7å”:˜j¨<Ñ7Ñ8Ô8ˆDÝÔ% d§k¢k¡m¤m°T´ZÑ@Ô@ˆCØ�OŠOØr˜˜Sœ	ÐrÐrÐr¨cÐrÐr¸
À3¼ÐrÐrÐrÕTYÐZfÐgjÔZkÑTlÔTlÐrÐrÐrñô ð ð �AŠvˆvÝ %Ô 3µE´LÀÐRVÐ4WÑ4WÔ4WÐYcÔYiÑ jÔ j�Ý"'Ô"5µe´lÀ<ÐVZÐ6[Ñ6[Ô6[Ð]iÔ]oÑ"pÔ"p�Ý"'Ô"3°MÀ?Ñ"SÔ"S�ùà )§¢µ°I±´Ñ ?Ô ?ÐàÝ�	‰NŒNØ!ØØð
ð 	
r    Úonnx_model_pathÚverboseÚuse_external_data_formatc
                 óŠ  — | j         }
|
j        }t                               ddd|
j        |
j        ||
j        |d|||||	¬¦  «        }|                     ¦   «         }t          j	        ¦   «         5   | |Ž }ddd¦  «         n# 1 swxY w Y   d„ t          |¦  «        D ¦   «         }d„ t          |¦  «        D ¦   «         }|d         j        d         |
j        k    s|d         j        d         |
j        k    sJ ‚|d         j        d         |
j        k    rd	nd
g|¢}ddddœ|d         dddœi}|D ]
}dddœ||<   Œ|D ]
}dddœ||<   Œdg}|rdddœ|d<   |                     d¦  «         |rdddœ|d<   |                     d¦  «         |                     |¦  «         t          |¦  «        dk    rt          |d         ¦  «        |k    sJ ‚t                               d|j        j        › d|j        d         j        › d|d         j        › d|d         d         j        › �¦  «         t'          |¦  «        j                             dd¬¦  «         |rÌt-          j        ¦   «         5 }t0          j                             |d¦  «        }t'          |¦  «        j                             dd¬¦  «         t7          | t9          |¦  «        |d|||ddd|¬¦  «         t;          j        |d¬¦  «        } t?          j         | |dd¬¦  «         ddd¦  «         dS # 1 swxY w Y   dS t7          | t9          |¦  «        |d|||ddd|¬¦  «         dS )z1Export GPT-2 model with past state to ONNX model.r7   F)r{   r|   r}   r~   r   rD   r€   r�   rw   r‚   rƒ   r„   r…   r†   Nc                 ó   — g | ]}d |› �‘ŒS )Úpast_rc   ©re   rF   s     r   rg   z*Gpt2Helper.export_onnx.<locals>.<listcomp>¥  s   € Ð<Ð<Ð< a�k˜a�k�kÐ<Ð<Ð<r    c                 ó   — g | ]}d |› �‘ŒS )r    rc   rð   s     r   rg   z*Gpt2Helper.export_onnx.<locals>.<listcomp>¦  s   € ÐBÐBÐB¨A˜ A˜˜ÐBÐBÐBr    r   r8   rZ   r[   r&   r{   Úseq_len)r   r7   Úpast_seq_len)r7   é   Útotal_seq_lenrJ   rK   zShapes: input_ids=z past=z output=z	 present=T)ÚparentsÚexist_okz	gpt2.onnxé   )
ÚargsÚfÚexport_paramsÚinput_namesÚoutput_namesÚdynamic_axesÚopset_versionÚdo_constant_foldingrì   rë   )Úload_external_data)Úsave_as_external_dataÚall_tensors_to_one_file)!r   rN   rz   r�   r~   r   r€   rj   r@   Úno_gradr>   r·   r?   rh   r=   rÊ   rÌ   r&   rP   r   ÚparentÚmkdirÚtempfileÚTemporaryDirectoryÚosÚpathÚjoinr
   r;   ÚonnxÚ
load_modelr   Úsave)Úmodelr�   rê   rë   rì   r‚   rƒ   r„   r…   r†   r   rD   Údummy_inputsri   ÚoutputsÚ
past_namesÚpresent_namesrý   rþ   r¶   rü   Útmp_dir_nameÚtemp_onnx_model_paths                          r   Úexport_onnxzGpt2Helper.export_onnx€  sÒ  € ð #œ\ˆØ”Nˆ	Ý!×2Ò2ØØ!"ØØ &Ô :ØÔ*ØØÔ(ØØØ-Ø1Ø+Ø1Ø!5ð 3ñ 
ô 
ˆð  "×)Ò)Ñ+Ô+ˆ
åŒ]‰_Œ_ð 	)ð 	)Ø�e˜ZÐ(ˆGð	)ð 	)ð 	)ñ 	)ô 	)ð 	)ð 	)ð 	)ð 	)ð 	)ð 	)øøøð 	)ð 	)ð 	)ð 	)ð =Ð<­5°Ñ+;Ô+;Ð<Ñ<Ô<ˆ
ØBÐBµ°yÑ1AÔ1AÐBÑBÔBˆð �qŒzÔ Ô" fÔ&7Ò7Ð7¸7À1¼:Ô;KÈAÔ;NÐRXÔRdÒ;dÐ;dÐ;dÐdØ$+¨A¤JÔ$4°QÔ$7¸6Ô;LÒ$LÐ$L˜˜ÐR^ÐoÐanÐoˆð ˜\¨iÐ8Ð8Ø˜ŒO °)Ð<Ð<ð
ˆð ð 	Fð 	FˆDØ%1°nÐ!EÐ!EˆL˜ÑÐØ!ð 	Gð 	GˆDØ%1°oÐ!FÐ!FˆL˜ÑÐà"�mˆØð 	/Ø/;À	Ð+JÐ+JˆL˜Ñ(Ø×Ò˜~Ñ.Ô.Ð.Øð 	1Ø1=À/Ð-RÐ-RˆLÐ)Ñ*Ø×ÒÐ/Ñ0Ô0Ð0Ø×Ò˜:Ñ&Ô&Ð&å�7‰|Œ|˜qÒ Ð ¥S¨°¬¡_¤_¸	Ò%AÐ%AÐ%AÐAå�Šð Z Ô!7Ô!=ð  Zð  ZÀ\ÔEVÐWXÔEYÔE_ð  Zð  ZÐipÐqrÔisÔiyð  Zð  Zð  ELð  MNô  EOð  PQô  ERô  EXð  Zð  Zñ	
ô 	
ð 	
õ 	ˆ_ÑÔÔ$×*Ò*°4À$Ð*ÑGÔGÐGà#ð (	åÔ,Ñ.Ô.ð °,Ý')¤w§|¢|°LÀ+Ñ'NÔ'NÐ$ÝÐ)Ñ*Ô*Ô1×7Ò7ÀÈtÐ7ÑTÔTÐTå!ØÝ˜zÑ*Ô*Ø*Ø"&Ø +Ø!-Ø!-Ø"$Ø(,Ø-1Ø#ðñ ô ð õ œÐ(<ÐQUÐVÑVÔV�Ý”ØØ#Ø*.Ø,0ð	ñ ô ð ð'ð ð ñ ô ð ð ð ð ð ð ð øøøð ð ð ð ð ð õ4 ØÝ˜:Ñ&Ô&Ø!Ø"Ø'Ø)Ø)Ø Ø$(Ø).Øðñ ô ð ð ð s%   Á-A?Á?BÂBÉ#BLÌLÌLr   c           	      óê   — t          d¦  «        }	t          | d||d|	d¬¦  «        }
|r5|rt                               |
¦  «         nd|vrd|d<    |
j        dddi|¤Ž |
                     ||¦  «         |
S )	zHOptimize ONNX model with an option to convert it to use mixed precision.r   r   F)Ú
model_typeÚ	num_headsr   Ú	opt_levelÚoptimization_optionsÚuse_gpuÚkeep_io_typesÚuse_symbolic_shape_inferTrc   )r   r	   rz   Úauto_mixed_precisionÚconvert_float_to_float16Úsave_model_to_file)rê   Úoptimized_model_pathr´   r~   r   rì   r  ÚstagerÍ   r  Úms              r   Úoptimize_onnxzGpt2Helper.optimize_onnxø  s¶   € õ  -¨VÑ4Ô4ÐåØØØ)Ø#ØØ!5Øð
ñ 
ô 
ˆð ð 	TØ#ð TÝ×/Ò/°Ñ2Ô2Ð2Ð2à"¨&Ð0Ð0Ø.3�F˜?Ñ+Ø*�Ô*ÐSÐSÀDÐSÈFÐSÐSÐSà	×ÒÐ1Ð3KÑLÔLÐLØˆr    )ÚAddÚLayerNormalizationÚSkipLayerNormalizationÚFastGeluÚEmbedLayerNormalizationÚ
onnx_modelÚop_block_listc                 ó\  — d„ |                       ¦   «         D ¦   «         }t          |¦  «        }|                     |¦  «        }t                               d|› d|› �¦  «         |                      ¦   «         j        d         j        }d}|                      ¦   «         }||v sJ ‚||         }d}	|j	        dk    r„|}	t                               d|j        › �¦  «         d}
|j
        D ]}|                      |¦  «        }
|
� nŒt          |
¦  «        }t                               d	|j        › d
|› �¦  «         |dk     }n*t                               d|j	        › d|j        › �¦  «         g }g }|s|	�|g}|	j        g}||||dœ}t                               d|› �¦  «          | j        dddi|¤Ž |S )a?  Convert GPT-2 model to mixed precision.
           It detects whether original model has fp16 weights, and set parameters for float16 conversion automatically.
        Args:
            onnx_model (OnnxModel): optimized ONNX model
            op_block_list (List[str], optional): operators to compute in fp32. Defaults to ["Add", "LayerNormalization",
                                                 "SkipLayerNormalization", "FastGelu", "EmbedLayerNormalization"]
        Returns:
            parameters(dict): a dictionary of parameters used in float16 conversion
        c                 ó   — h | ]	}|j         ’Œ
S rc   )Úop_type)re   Únodes     r   ú	<setcomp>z2Gpt2Helper.auto_mixed_precision.<locals>.<setcomp>0  s   € ÐCÐCÐC¨�t”|ÐCÐCÐCr    z	fp32 op: z
 fp16 op: r   FNÚMatMulz#Found last MatMul node for logits: z3max diff of converting weights in last MatMul node ú: rº   z-Failed to find MatMul node for logits. Found z	 of node )r  r,  Únode_block_listÚforce_fp16_initializersz!auto_mixed_precision parameters: r  Trc   )ÚnodesÚsetÚ
differencerÊ   rÌ   ÚgraphÚoutputr¶   Úoutput_name_to_noder/  ÚinputÚget_initializerr   rË   Úwarningr   )r+  r,  Úop_full_setÚfp32_op_setÚfp16_op_setÚlogits_output_nameÚis_weight_fp16_precisionr;  r0  Úlast_matmul_nodeÚinitializerr<  Úmax_diffr  r4  Ú
parameterss                   r   r  zGpt2Helper.auto_mixed_precision  s  € ð( DÐC°
×0@Ò0@Ñ0BÔ0BÐCÑCÔCˆÝ˜-Ñ(Ô(ˆØ!×,Ò,¨[Ñ9Ô9ˆÝ�ŠÐD ÐDÐD°{ÐDÐDÑEÔEÐEð (×-Ò-Ñ/Ô/Ô6°qÔ9Ô>Ðð $)Ð Ø(×<Ò<Ñ>Ô>ÐØ!Ð%8Ð8Ð8Ð8Ð8Ø"Ð#5Ô6ˆØÐØŒ<˜8Ò#Ð#Ø#ÐÝ�KŠKÐI¸d¼iÐIÐIÑJÔJÐJØˆKØœð ð �Ø(×8Ò8¸Ñ?Ô?�ØÐ*Ø�Eð +õ
 1°Ñ=Ô=ˆHÝ�LŠLÐfÈtÌyÐfÐfÐ\dÐfÐfÑgÔgÐgØ'/°$¢Ð$Ð$å�NŠNÐmÈ4Ì<ÐmÐmÐbfÔbkÐmÐmÑnÔnÐnàˆØˆØ(ð 	6Ð/?Ð/Kà/Ð0ˆMØ/Ô4Ð5ˆOð +Ø*Ø.Ø'?ð	
ð 
ˆ
õ 	�ŠÐD¸
ÐDÐDÑEÔEÐEØ+ˆ
Ô+ÐXÐXÀTÐXÈZÐXÐXÐXàÐr    ÚinputsÚ
total_runsc                 ó¶  — t                                d¦  «         |                     ¦   «                              ¦   «         }t	          j        ¦   «         5   | |Ž }ddd¦  «         n# 1 swxY w Y   |dk    r|S g }t	          j        ¦   «         5  t          |¦  «        D ]C}t          j        ¦   «         } | |Ž }|                     t          j        ¦   «         |z
  ¦  «         ŒD	 ddd¦  «         n# 1 swxY w Y   t          |¦  «        dz  t          |¦  «        z  }t                                d                     t          |d¦  «        ¦  «        ¦  «         ||fS )zfRun inference of PyTorch model, and returns average latency in ms when total_runs > 0 besides outputs.zstart pytorch_inferenceNr   éè  zPyTorch inference time = {} msú.2f)rÊ   rË   rx   rj   r@   r  r>   Útimer?   Úsumr=   Úformat)	r  rH  rI  ri   r  Úlatencyr�   ÚstartÚaverage_latencys	            r   Úpytorch_inferencezGpt2Helper.pytorch_inferenceb  s¾  € õ 	�ŠÐ.Ñ/Ô/Ð/ð —^’^Ñ%Ô%×-Ò-Ñ/Ô/ˆ
åŒ]‰_Œ_ð 	)ð 	)Ø�e˜ZÐ(ˆGð	)ð 	)ð 	)ñ 	)ô 	)ð 	)ð 	)ð 	)ð 	)ð 	)ð 	)øøøð 	)ð 	)ð 	)ð 	)ð ˜Š?ˆ?ØˆNàˆÝŒ]‰_Œ_ð 	4ð 	4Ý˜:Ñ&Ô&ð 4ð 4�Ýœ	™œ�Ø˜% Ð,�Ø—’�tœy™{œ{¨UÑ2Ñ3Ô3Ð3Ð3ð4ð	4ð 	4ð 	4ñ 	4ô 	4ð 	4ð 	4ð 	4ð 	4ð 	4ð 	4øøøð 	4ð 	4ð 	4ð 	4õ ˜g™,œ,¨Ñ-µ°G±´Ñ<ˆÝ�ŠÐ5×<Ò<½VÀOÐUZÑ=[Ô=[Ñ\Ô\Ñ]Ô]Ð]à˜Ð'Ð's%   ÁA&Á&A*Á-A*ÂAC/Ã/C3Ã6C3c                 óX  — t                                d¦  «         dt          j        |j                             ¦   «                              ¦   «         ¦  «        i}|j        �Xt          |j        ¦  «        D ]C\  }}t          j        |                     ¦   «                              ¦   «         ¦  «        |d|› �<   ŒD|j        �@t          j        |j                             ¦   «                              ¦   «         ¦  «        |d<   |j	        �@t          j        |j	                             ¦   «                              ¦   «         ¦  «        |d<   |  
                    d|¦  «        }|dk    r|S g }t          |¦  «        D ]T}t          j        ¦   «         }	|  
                    d|¦  «        }|                     t          j        ¦   «         |	z
  ¦  «         ŒUt          |¦  «        dz  t          |¦  «        z  }
t                                d	                     t!          |
d
¦  «        ¦  «        ¦  «         ||
fS )zcRun inference of ONNX model, and returns average latency in ms when total_runs > 0 besides outputs.zstart onnxruntime_inferencer&   Nrï   rK   rJ   r   rK  z"OnnxRuntime Inference time = {} msrL  )rÊ   rË   rª   Úascontiguousarrayr&   r»   rP   Ú	enumeraterK   rJ   Úrunr>   rM  r?   rN  r=   rO  )Úort_sessionrH  rI  Ú
ort_inputsrF   Úpast_ir¿   rP  r�   rQ  rR  s              r   Úonnxruntime_inferencez Gpt2Helper.onnxruntime_inference|  sá  € õ 	�ŠÐ2Ñ3Ô3Ð3à!¥5Ô#:¸6Ô;K×;OÒ;OÑ;QÔ;Q×;WÒ;WÑ;YÔ;YÑ#ZÔ#ZÐ[ˆ
àŒ;Ð"Ý& v¤{Ñ3Ô3ð Xð X‘	��6Ý*/Ô*AÀ&Ç*Â*Á,Ä,×BTÒBTÑBVÔBVÑ*WÔ*W�
˜; 1˜;˜;Ñ'Ð'àÔ Ð,Ý+0Ô+BÀ6ÔCX×C\ÒC\ÑC^ÔC^×CdÒCdÑCfÔCfÑ+gÔ+gˆJÐ'Ñ(àÔÐ*Ý).Ô)@ÀÔAT×AXÒAXÑAZÔAZ×A`ÒA`ÑAbÔAbÑ)cÔ)cˆJ�~Ñ&à!—o’o d¨JÑ7Ô7ˆØ˜Š?ˆ?ØÐàˆÝ�zÑ"Ô"ð 	0ð 	0ˆAÝ”I‘K”KˆEØ%Ÿ/š/¨$°
Ñ;Ô;ˆKØ�NŠN�4œ9™;œ;¨Ñ.Ñ/Ô/Ð/Ð/å˜g™,œ,¨Ñ-µ°G±´Ñ<ˆÝ�ŠÐ9×@Ò@ÅÈÐY^ÑA_ÔA_Ñ`Ô`ÑaÔaÐaà˜OÐ+Ð+r    c           	      ó6   — t          j        | ||||||¦  «        S )z)Returnas IO binding object for a session.)r   Úprepare_io_binding)rX  r&   rJ   rK   rP   r®   r§   s          r   r]  zGpt2Helper.prepare_io_bindingœ  s.   € õ Ô1ØØØØØØØñ
ô 
ð 	
r    c                 ó0   — t          j        | |||¦  «        S )z3Copy results to cpu. Returns a list of numpy array.)r   Ú"get_outputs_from_io_binding_buffer)rX  r®   r§   Úreturn_numpys       r   r_  z-Gpt2Helper.get_outputs_from_io_binding_buffer±  s"   € õ ÔAØ˜¨¸ñ
ô 
ð 	
r    r®   r§   r`  Úinclude_copy_output_latencyc           	      óœ  — t                                d¦  «         t                               | |j        |j        |j        |j        ||¦  «        }|                      |¦  «         t           	                    | |||¦  «        }|dk    r|S g }	t          |¦  «        D ]r}
t          j        ¦   «         }|                      |¦  «         |rt           	                    | |||¦  «        }
|	                     t          j        ¦   «         |z
  ¦  «         Œst          |	¦  «        dz  t          |	¦  «        z  }t                                d|¦  «         ||fS )zUInference with IO binding. Returns outputs, and optional latency when total_runs > 0.z*start onnxruntime_inference_with_binded_ior   rK  z4OnnxRuntime with IO binding inference time = %.2f ms)rÊ   rË   rz   r]  r&   rJ   rK   rP   Úrun_with_iobindingr_  r>   rM  r?   rN  r=   )rX  rH  r®   r§   rI  r`  ra  Ú
io_bindingr¿   rP  r�   rQ  rR  s                r   Ú$onnxruntime_inference_with_binded_ioz/Gpt2Helper.onnxruntime_inference_with_binded_io¸  sS  € õ 	�ŠÐAÑBÔBÐBõ  ×2Ò2ØØÔØÔØÔ!ØŒKØØñ
ô 
ˆ
ð 	×&Ò& zÑ2Ô2Ð2õ !×CÒCØ˜¨¸ñ
ô 
ˆð ˜Š?ˆ?ØÐàˆÝ�zÑ"Ô"ð 	0ð 	0ˆAÝ”I‘K”KˆEà×*Ò*¨:Ñ6Ô6Ð6Ø*ð Ý×AÒAØ °Àñô �ð �NŠN�4œ9™;œ;¨Ñ.Ñ/Ô/Ð/Ð/å˜g™,œ,¨Ñ-µ°G±´Ñ<ˆÝ�ŠÐKÈ_Ñ]Ô]Ð]à˜OÐ+Ð+r    c                 ó‚  — t          d| › d�d¦  «        5 }t          j        ||¦  «         d d d ¦  «         n# 1 swxY w Y   t                               d| › d�¦  «         t          d| › d�d¦  «        5 }t          j        ||¦  «         d d d ¦  «         n# 1 swxY w Y   t                               d| › d�¦  «         d S )NÚort_outputs_ú.pickleÚwbz$ORT output are saved to ort_outputs_Útorch_outputs_z(Torch output are saved to torch_outputs_©ÚopenÚpickleÚdumprÊ   rÌ   )rF   r¿   r¾   rú   s       r   Úsave_outputszGpt2Helper.save_outputsë  sD  € åÐ+ Ð+Ð+Ð+¨TÑ2Ô2ð 	(°aÝŒK˜ QÑ'Ô'Ð'ð	(ð 	(ð 	(ñ 	(ô 	(ð 	(ð 	(ð 	(ð 	(ð 	(ð 	(øøøð 	(ð 	(ð 	(ð 	(å�ŠÐE¸1ÐEÐEÐEÑFÔFÐFåÐ- 1Ð-Ð-Ð-¨tÑ4Ô4ð 	*¸ÝŒK˜ qÑ)Ô)Ð)ð	*ð 	*ð 	*ñ 	*ô 	*ð 	*ð 	*ð 	*ð 	*ð 	*ð 	*øøøð 	*ð 	*ð 	*ð 	*å�ŠÐI¸qÐIÐIÐIÑJÔJÐJÐJÐJs   •7·;¾;Á4BÂBÂBc                 óÄ   — t          d| › d�d¦  «        5 }t          j        ||¦  «         d d d ¦  «         n# 1 swxY w Y   t                               d| › d�¦  «         d S )NÚdummy_inputs_rh  ri  z!inputs are saved to dummy_inputs_rk  )rF   r  r¿   r¾   rú   s        r   Úsave_inputszGpt2Helper.save_inputsõ  s¦   € åÐ, !Ð,Ð,Ð,¨dÑ3Ô3ð 	)°qÝŒK˜ aÑ(Ô(Ð(ð	)ð 	)ð 	)ñ 	)ô 	)ð 	)ð 	)ð 	)ð 	)ð 	)ð 	)øøøð 	)ð 	)ð 	)ð 	)å�ŠÐB¸ÐBÐBÐBÑCÔCÐCÐCÐCs   •7·;¾;r   i'  r7   c                 ó¾  ‡‡,— |j         }t                               d|› d‰› d|› d|› d|	› d|› d�¦  «         d}d	}d
}d}|r:t                               |||||	¦  «        }t                               |||¦  «        }d}d}g Š,dg|z  }‰|z  }t          |¦  «        D �]p}t          |‰z  ¦  «        }t          j	        d|¦  «        }|dk    rdnt          j	        d|¦  «        }t          j	        d|¦  «        } t           
                    d| › d|› d�¦  «         t                               | |||j        |j        |j        |j        |||
||||d¬¦  «        }!t                               ||!¦  «        }"|rt                               | |!¦  «        }#n;t                               | ||||	¦  «        }$t                               | |!||$¦  «        }#t                               |"|#|¬¦  «        \  }%}&}'}(})t)          j        |&¦  «        s‰,                     |&¦  «         |%r|dz  }|)r|dz  }||xx         dz  cc<   |r€|%s~t                               d|› d| › d|› d|› d|&› �
¦  «         t/          |(¦  «        D ]E\  }}*t                               d|› d|                      ¦   «         |         j        › d|*› �¦  «         ŒF|rTt)          j        |&¦  «        s	|&d|z  k    r7t                               ||!¦  «         t                               ||#|"¦  «         �Œr‰,rˆ,fd„dD ¦   «         }+nd„ dD ¦   «         }+|dz  |z  |+d <   ˆfd!„|D ¦   «         |+d"<   |dz  |z  |+d#<   |t9          ‰,¦  «        z
  dz  |z  |+d$<   t                               d%|› d&|› d'|t9          ‰,¦  «        z
  › d(|› �¦  «         |d)|z  k    r2t                               d*t          |dz  |z  ¦  «        d+›d,�¦  «         |+S )-zKGenerate random inputs and compare the results of PyTorch and Onnx Runtime.zRunning parity test (atol=z, test_cases=z, runs=z, use_io_binding=z, model_class=z, is_float16=z) ...é   é   r8   Nr   r7   z#Running parity test for batch_size=z past_sequence_length=z...T)r„   r…   r†   r‡   )rÇ   z
test_case=z batch_size=z sequence_length=z	 MaxDiff=ú	z: Name=z, éd   c                 óF   •— i | ]}d |› �t          j        ‰|¦  «        d›“ŒS )Úmax_diff_percentile_rÈ   )rª   Ú
percentile)re   rv   Úmax_abs_diff_lists     €r   ú
<dictcomp>z*Gpt2Helper.test_parity.<locals>.<dictcomp>o  sF   ø€ ð ð ð Ø`aÐ* qÐ*Ð*­uÔ/?Ð@QÐSTÑ/UÔ/UÐ,[Ð,[ðð ð r    )é2   éZ   é_   éc   c                 ó   — i | ]}d |› �d“Œ	S )ry  Únanrc   ru   s     r   r|  z*Gpt2Helper.test_parity.<locals>.<dictcomp>s  s#   € ÐRÐRÐR¸AÐ0¨QÐ0Ð0°%ÐRÐRÐRr    r‹   Útop1_match_ratec                 ó    •— g | ]
}|d z  ‰z  ‘ŒS )r‹   rc   )re   ÚxÚtest_cases_per_runs     €r   rg   z*Gpt2Helper.test_parity.<locals>.<listcomp>v  s#   ø€ Ð,nÐ,nÐ,nÈa¨Q°©WÐ7IÑ-IÐ,nÐ,nÐ,nr    Útop1_match_rate_per_runÚdiff_pass_rateÚnan_ratezParity Test Cases=z	; Passed=z; Nan=z; Top1_Matched=gffffffî?zParity is good: passed rate=z.0fú%)r   rÊ   rÌ   rz   r¨   r¸   r>   r”   r—   r•   rË   r�   r~   r   rN   r€   rS  r[  re  ré   rª   rÖ   r?   rV  Úget_outputsr¶   rr  ro  r=   )-rX  r  r�   r´   rÆ   rÇ   r†  rI  Úuse_io_bindingrž   r‚   rƒ   r„   r…   r†   r#  rë   Úenable_pickle_outputr   Úmax_batch_sizeÚmax_past_seq_lenÚmax_seq_lenr®   Úmax_output_shapesÚpassed_test_casesÚtop1_matched_casesÚtop1_matched_cases_per_runÚtotal_test_casesrF   Úrun_idr}   r|   r{   r  r  r¿   r§   rÏ   rÒ   rè   râ   rà   ÚmessagerC   r{  s-         `                                     @r   Útest_parityzGpt2Helper.test_parityû  sÊ  øø€ ð. #œ\ˆå�Šð ~¨ð  ~ð  ~Ð<Nð  ~ð  ~ÐWað  ~ð  ~ð  uCð  ~ð  ~ð  S^ð  ~ð  ~ð  mwð  ~ð  ~ð  ~ñ	
ô 	
ð 	
ð ˆØÐØˆàˆØð 	bÝ *× <Ò <ØÐ 0°+¸vÀ{ñ!ô !Ðõ (×:Ò:Ð;LÈfÐV`ÑaÔaˆNàÐØÐàÐØ&' S¨:Ñ%5Ð"Ø-°
Ñ:ÐÝÐ'Ñ(Ô(ð C	Añ C	AˆAÝ˜Ð/Ñ/Ñ0Ô0ˆFÝ$œn¨Q°Ñ<Ô<ˆOØ).°!ª¨ 1 1½&¼.ÈÐL\Ñ:]Ô:]Ð Ýœ¨¨>Ñ:Ô:ˆJå�LŠLØq°jÐqÐqÐXlÐqÐqÐqñô ð õ &×6Ò6ØØ$ØØÔ*ØÔ"Ø”ØÔ!ØØØ Ø"Ø /Ø#5Ø%9Ø"&ð 7ñ ô ˆLõ" !×2Ò2°5¸,ÑGÔGˆGØð Ý(×>Ò>¸{ÈLÑYÔY��å *× <Ò <ØØ(Ø#ØØñ!ô !�õ )×MÒMØ ¨~¸}ñô �õ ×-Ò-¨g°{ÈÐ-ÑNÔNñØØØ%ØØå”;˜|Ñ,Ô,ð 7Ø!×(Ò(¨Ñ6Ô6Ð6Øð 'Ø! QÑ&Ð!àð 8Ø" aÑ'Ð"Ø*¨6Ð2Ð2Ô2°aÑ7Ð2Ð2Ñ2àð _˜|ð _Ý—’ð b ð  bð  b°
ð  bð  bÐRfð  bð  bð  zIð  bð  bð  T`ð  bð  bñô ð õ #,¨HÑ"5Ô"5ð _ð _‘J�A�wÝ—K’KÐ ] QÐ ]Ð ]¨{×/FÒ/FÑ/HÔ/HÈÔ/KÔ/PÐ ]Ð ]ÐT[Ð ]Ð ]Ñ^Ô^Ð^Ð^ð $ð A­¬°\Ñ)BÔ)Bð AÀlÐUXÐ[_ÑU_ÒF_ÐF_Ý×&Ò& q¨,Ñ7Ô7Ð7Ý×'Ò'¨¨;¸Ñ@Ô@Ð@ùàð 	Sðð ð ð Øeuðñ ô ˆFˆFð SÐRÐAQÐRÑRÔRˆFà$6¸Ñ$<Ð?OÑ$OˆÐ Ñ!Ø,nÐ,nÐ,nÐ,nÐSmÐ,nÑ,nÔ,nˆÐ(Ñ)Ø#4°sÑ#:Ð=MÑ#MˆÐÑ Ø.µÐ5FÑ1GÔ1GÑGÈ3ÑNÐQaÑaˆˆzÑå�Šð dÐ!1ð  dð  dÐ<Mð  dð  dÐUeÕhkÐl}Ñh~Ôh~ÑU~ð  dð  dð  Pbð  dð  dñ	
ô 	
ð 	
ð ˜tÐ&6Ñ6Ò6Ð6Ý�KŠKÐmµsÐ;LÈsÑ;RÐUeÑ;eÑ7fÔ7fÐmÐmÐmÐmÑnÔnÐnàˆr    rw  rt  é    c                 óŽ  — |j         }d}|r:t                               |||||¦  «        }t                               |||¦  «        }t                               ||||j        |j        |j        |j        |||||	|
|¬¦  «        }|r t           	                    | ||¦  «        \  }}n!t           
                    | ||||¦  «        \  }}|S )zCGenerate random inputs and measure average latency of Onnx Runtime.N)r„   r…   r†   )r   rz   r¨   r¸   r�   r~   r   rN   r€   r[  re  )rX  r  r�   r´   rI  rŒ  rž   r‚   rƒ   r„   r…   r†   r{   r}   r|   r   r®   r§   r  r�   rP  s                        r   Útest_performancezGpt2Helper.test_performanceƒ  sö   € ð( #œ\ˆàˆØð 	^Ý&×8Ò8ØÐ0°/À6È;ñô ˆMõ (×:Ò:¸=È&ÐR\Ñ]Ô]ˆNå!×2Ò2ØØ ØØÔ&ØÔØŒNØÔØØØØØ+Ø1Ø!5ð 3ñ 
ô 
ˆð" ð 	Ý#×9Ò9¸+À|ÐU_Ñ`Ô`‰JˆAˆwˆwå#×HÒHØ˜\¨>¸=È*ñô ‰JˆAˆwð ˆr    c                 óØ   — t                                ddd|j        |j        |j        |j        |d||¬¦  «                             ¦   «         }t          j         	                    | |¦  «        S )zJIT trace for TorchScript.r7   F)r{   r|   r}   r~   r   rD   r€   r�   rw   r‚   rƒ   )
rz   r�   r~   r   rN   r€   rj   r@   ÚjitÚtrace)r  r   r�   r‚   rƒ   ri   s         r   ÚtorchscriptzGpt2Helper.torchscriptº  sp   € õ  ×0Ò0ØØ!"ØØ &Ô :ØÔ*Ø”nØÔ(ØØØ-Ø1ð 1ñ 
ô 
÷ Š'‰)Œ)ð 	õ Œy�Š˜u jÑ1Ô1Ð1r    ©ÚrawÚfp32Úfp16Úint8c           
      óÜ  — |}t           j                             |¦  «        rt          |¦  «        j        d         }n|                     d¦  «        d          |dk    r|d|z   z  }|r|dz  }|�ródddd	d
œ}d
D ]Ý}t           j                             | |||         z   ¦  «        }	t           j                             |	¦  «        r“||v ro	 t          j	        |	¦  «         t                               d|	› �¦  «         Œ�# t          $ r/}
t                               d|	› d|
j        › �¦  «         Y d}
~
Œµd}
~
ww xY wt                               d|› d|	› �¦  «         ŒÞt           j                             t           j                             | |¦  «        |dz   ¦  «        t           j                             t           j                             | |dz   ¦  «        |dz   ¦  «        t           j                             t           j                             | |dz   ¦  «        |dz   ¦  «        t           j                             t           j                             | |d	z   ¦  «        |dz   ¦  «        d
œS t           j                             | |dz   ¦  «        t           j                             | |dz   ¦  «        t           j                             | |dz   ¦  «        t           j                             | |dz   ¦  «        d
œS )z=Build a  path name for given model based on given attributes.r“   ú/r   r�   Ú_pastÚ Ú_fp32Ú_fp16Ú_int8r   zRemoved the existed directory: zFailed to remove the directory r3  NzDirectory for z
 existed: z.onnxz
_fp32.onnxz
_fp16.onnxz
_int8.onnx)r	  r
  Úisdirr   ÚpartsÚsplitr  ÚexistsÚshutilÚrmtreerÊ   rÌ   ÚOSErrorÚstrerror)Ú
output_dirÚmodel_name_or_pathrž   Úhas_pastÚ
new_folderÚremove_existingÚ
model_nameÚsuffixr  Únew_dirÚes              r   Úget_onnx_pathszGpt2Helper.get_onnx_pathsÌ  sä  € ð (ˆ
ÝŒ7�=Š=Ð+Ñ,Ô,ð 	&ÝÐ0Ñ1Ô1Ô7¸Ô;ˆJˆJà×Ò˜SÑ!Ô! "Ô%Ð%àÐ+Ò+Ð+Ø˜# Ñ+Ñ+ˆJàð 	"Ø˜'Ñ!ˆJàñ 	Ø¨¸'È7ÐSÐSˆFà=ð 
Vð 
V�
Ýœ'Ÿ,š, z°:ÀÀzÔ@RÑ3RÑSÔS�Ý”7—>’> 'Ñ*Ô*ð VØ! _Ð4Ð4ðcÝ"œM¨'Ñ2Ô2Ð2Ý"ŸKšKÐ(SÈ'Ð(SÐ(SÑTÔTÐTÐTøÝ&ð cð cð cÝ"ŸKšKÐ(aÈ'Ð(aÐ(aÐUVÔU_Ð(aÐ(aÑbÔbÐbÐbÐbÐbÐbÐbøøøøðcøøøõ ŸšÐ$T°ZÐ$TÐ$TÈ7Ð$TÐ$TÑUÔUÐUøõ ”w—|’|¥B¤G§L¢L°¸ZÑ$HÔ$HÈ*ÐW^ÑJ^Ñ_Ô_ÝœŸšÝ”G—L’L ¨Z¸'Ñ-AÑBÔBØ Ñ-ñô õ œŸšÝ”G—L’L ¨Z¸'Ñ-AÑBÔBØ Ñ-ñô õ œŸšÝ”G—L’L ¨Z¸'Ñ-AÑBÔBØ Ñ-ñô ðð ð õ" ”7—<’< 
¨J¸Ñ,@ÑAÔAÝ”G—L’L ¨Z¸,Ñ-FÑGÔGÝ”G—L’L ¨Z¸,Ñ-FÑGÔGÝ”G—L’L ¨Z¸,Ñ-FÑGÔGð	
ð 
ð 	
s   Ã1C:Ã:
D3Ä%D.Ä.D3N)r   )F)rÄ   rÄ   )rº   )FFr   )r   )T)r   TF)TT)'r(   r)   r*   r+   rQ   r@   Úint32r”   r�   Úboolrq   r^   r�   r   r£   Údictr<   r¨   r±   r¸   rÃ   rÓ   ré   r  r%  r   r  rS  r[  r]  r_  ÚTensorre  ro  rr  r˜  r›  rŸ  r½  rc   r    r   rz   rz   ¨   s­  € € € € € ØOÐOàð Ø!%Ø#'Ø',¤{Ø*/¬+Ø,1¬KØ"&ð>Ið >IØð>Ià!ð>Ið ð>Ið !ð	>Ið
 ð>Ið ð>Ið ð>Ið ”ð>Ið ð>Ið ð>Ið !ð>Ið œð>Ið "œKð>Ið $œkð>Ið  ð>Ið  
ð!>Ið >Ið >Iñ „\ð>Ið@ ð -ð ð  Øð à!ð ð ð ð ð	 ð
 ð ð 
ˆc�4˜”9ˆnÔ	ð ð  ð  ñ „\ð ðD ð	ð 	ñ „\ð	ð ðð ð ñ „\ðð ð$ð $ð $ñ „\ð$ð ðð ð ñ „\ðð4 ð3
ð 3
ð 3
ñ „\ð3
ðj ð
 Ø).Ø!%Ø#'Ø',¤{Ø*/¬+Ø,1¬Kðuð uð ðuð ð	uð
 #'ðuð ðuð !ðuð œðuð "œKðuð $œkðuð uð uñ „\ðuðn ð "'Ø"Øð!ð !ð !ñ „\ð!ðF ð$
ð $
ð $
ðCð CØðCà˜C”yðCð Cð Cñ „\ðCðJ ð(ð (¨ð (Àð (ð (ð (ñ „\ð(ð2 ð,ð ,°:ð ,È3ð ,ð ,ð ,ñ „\ð,ð> ð
ð 
ñ „\ð
ð( ð
ð 
ð 
ñ „\ð
ð ð Ø!Ø,1ð0,ð 0,àð0,ð ˜S %¤,Ð.Ô/ð0,ð ˜C  c¤˜NÔ+ð	0,ð
 ð0,ð ð0,ð &*ð0,ð 0,ð 0,ñ „\ð0,ðd ðKð Kñ „\ðKð ðDð Dñ „\ðDð
 ð
 ØØØ ØØØ%ØØØœØ œ;Ø"œ[ØØØ"ð%Eð Eð Eñ „\ðEðN ð
 ØØØ%ØØØœØ œ;Ø"œ[ØØØð4ð 4ð 4ñ „\ð4ðl ð2ð 2ð 2ñ „\ð2ð" ð -ØØØ7Ð7Ð7ð:
ð :
ð ð:
ð :
ð :
ñ „\ð:
ð :
ð :
r    rz   )/Úloggingr	  rm  r—   r°  r  rM  Úpathlibr   rª   r  r@   Úbenchmark_helperr   rw   r   Úfusion_optionsr   Úio_binding_helperr   r+  r   Ú	optimizerr	   Útorch_onnx_export_helperr
   Útransformersr   r   r   r   Ú	getLoggerr(   rÊ   ÚPRETRAINED_GPT2_MODELSÚFLOAT32ÚFLOAT16ÚINT8ÚDEFAULT_TOLERANCEr   r/   r4   rS   rW   r¢   r^   rz   rc   r    r   ú<module>rÐ     sß  ðð €€€Ø 	€	€	€	Ø €€€Ø €€€Ø €€€Ø €€€Ø €€€Ø Ð Ð Ð Ð Ð à €€€Ø €€€Ø €€€Ø &Ð &Ð &Ð &Ð &Ð &Ø -Ð -Ð -Ð -Ð -Ð -Ø (Ð (Ð (Ð (Ð (Ð (Ø -Ð -Ð -Ð -Ð -Ð -Ø  Ð  Ð  Ð  Ð  Ð  Ø $Ð $Ð $Ð $Ð $Ð $Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø LÐ LÐ LÐ LÐ LÐ LÐ LÐ LÐ LÐ LÐ LÐ Là	ˆÔ	˜8Ñ	$Ô	$€àWÐWÐWÐ ð Ô�vØÔ�sØ„N�CðÐ ðNð Nð Nð Nð N˜9ñ Nô Nð Nð8ð 8ð 8ð 8ð 8˜[ñ 8ô 8ð 8ð!Eð !Eð !Eð !Eð !E�)ñ !Eô !Eð !EðHEð Eð Eð Eð E˜ñ Eô Eð Eð$Eð Eð Eð Eð E /ñ Eô Eð Eð" *¨8°TÐ:Ø"=¸xÈÐ!OØ˜|¨TÐ2ðð €ðSð Sð Sð Sð Sñ Sô Sð Sð>_
ð _
ð _
ð _
ð _
ñ _
ô _
ð _
ð _
ð _
r    