§
    û¸¸j6E  ã                   ó$  — d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	m
Z
mZ d dlmZ d dlmZmZ d dlmZ  ej        d¦  «        Zd'd„Z G d„ d	¦  «        Zd
„ Zd„ Zd„ Zd„ Zd„ Zdefd„Zd„ Zddgfdefd„Zd„ Z d„ Z!d„ Z"d„ Z#defd„Z$e%dk    �rw e¦   «         Z& ee&j'        ¦  «         e&j(        dk     se&j)        dk     se&j(        e&j)        z  dk     re *                    d¦  «         ej+         ,                    e&j        ¦  «        rRe&j-        sKe&j.        s e/d e&j        › d!�¦  «        ‚e 0                    d"e&j        ¦  «          ej1        e&j        ¦  «          ee&j(        e&j)        e&j        ¦  «        Z2e&j-        s e$e2e&¦  «         	  ee2j3        ¦  «        Z4n/# e5$ r' e 6                    d#e2j3        › �¦  «         e2j7        Z4Y nw xY we 0                    d$¦  «         e2j3         8                    d%d&¦  «        Z9 ee4e9¦  «         dS dS )(é    N)Úget_ort_environment_variablesÚsetup_logger)Úmain)ÚPRETRAINED_GPT2_MODELSÚ
Gpt2Helper)Ú	OnnxModelÚ c           
      ó¶  — t          j        ¦   «         }|                     dddt          dd                     t
          ¦  «        z   ¬¦  «         |                     ddt          d	d
¬¦  «         |                     ddt          dd¬¦  «         |                     ddt          dd¬¦  «         |                     dddd¬¦  «         |                     d¬¦  «         |                     dddd¬¦  «         |                     d¬¦  «         |                     dddd¬¦  «         |                     d¬¦  «         |                     ddd¬¦  «         |                     d¬¦  «         |                     d ddd!¬¦  «         |                     d¬"¦  «         |                     d#ddd$¬¦  «         |                     d¬%¦  «         |                     | ¦  «        }|S )&Nz-mz--model_name_or_pathTz2Model path, or pretrained model name in the list: z, )ÚrequiredÚtypeÚhelpz--csvFzgpt2_parity_results.csvz#path of csv file to save the result)r   r   Údefaultr   z--test_casesiô  znumber of test cases per runz--runsé(   znumber of repeated runsú	--use_gpuÚ
store_truezuse GPU for inference)r   Úactionr   )Úuse_gpuz--allz'run all combinations of mixed precision)Úallz-eú--use_external_data_format)r   r   )Úuse_external_data_formatz	--verbose)Úverbosez--skip_testzEdo not run test, and only rank experiments based on existing csv file)Ú	skip_testz--overwritezOverwrite existing csv file)Ú	overwrite)	ÚargparseÚArgumentParserÚadd_argumentÚstrÚjoinr   ÚintÚset_defaultsÚ
parse_args)ÚargvÚparserÚargss      ú€/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/onnxruntime/transformers/models/gpt2/gpt2_parity.pyÚparse_argumentsr&      sG  € ÝÔ$Ñ&Ô&€Fà
×ÒØØØÝØAÀDÇIÂIÕNdÑDeÔDeÑeð ñ ô ð ð ×ÒØØÝØ)Ø2ð ñ ô ð ð ×ÒØØÝØØ+ð ñ ô ð ð ×Ò˜¨5µsÀBÐMfÐÑgÔgÐgà
×Ò˜¨e¸LÐOfÐÑgÔgÐgØ
×Ò ÐÑ&Ô&Ð&à
×ÒØØØØ6ð	 ñ ô ð ð ×Ò˜EÐÑ"Ô"Ð"à
×Ò˜Ð:ÀUÐS_ÐÑ`Ô`Ð`Ø
×Ò°ÐÑ7Ô7Ð7à
×Ò˜¨e¸LÐÑIÔIÐIØ
×Ò ÐÑ&Ô&Ð&à
×ÒØØØØTð	 ñ ô ð ð ×Ò %ÐÑ(Ô(Ð(à
×ÒØØØØ*ð	 ñ ô ð ð ×Ò %ÐÑ(Ô(Ð(à×Ò˜TÑ"Ô"€Dà€Kó    c                   ó   — e Zd Zd„ Zd„ ZdS )Ú
ParityTaskc                 óL   — || _         || _        || _        g | _        d| _        d S )Nr   )Ú
total_runsÚ
test_casesÚcsv_pathÚresultsÚrun_id)Úselfr,   r+   r-   s       r%   Ú__init__zParityTask.__init__b   s)   € Ø$ˆŒØ$ˆŒØ ˆŒØˆŒØˆŒˆˆr'   c                 ó–  — t           j                              ¦   «                              d¦  «        }|› d| j        › �}| xj        dz  c_        	 t	          g |¢d‘| j        › ‘d‘| j        › ‘||| j        ¬¦  «        }|r| j         	                    |¦  «         n/# t          $ r" t                               d|› �¦  «         d }Y nw xY w|S )Nz%Y%m%d%H%M%SÚ_é   z-tz-r)Úexperiment_namer/   Úcsv_filenamezFailed to run experiment )ÚdatetimeÚnowÚstrftimer/   r   r,   r+   r-   r.   ÚappendÚ	ExceptionÚloggerÚ	exception)r0   r"   r5   Ú
start_timer/   Úresults         r%   ÚrunzParityTask.runi   sü   € ÝÔ&×*Ò*Ñ,Ô,×5Ò5°nÑEÔEˆ
ØÐ.Ð. ¤Ð.Ð.ˆØˆŒ�qÑˆŒð	ÝØO�$ÐO˜ÐO ¤Ð2ÐO°DÐO¸T¼_Ð:NÐOØ /ØØ!œ]ð	ñ ô ˆFð ð ,Ø”×#Ò# FÑ+Ô+Ð+øøÝð 	ð 	ð 	Ý×ÒÐJ¸ÐJÐJÑKÔKÐKØˆFˆFˆFð	øøøð ˆs   ÁA
B Â)CÃCN)Ú__name__Ú
__module__Ú__qualname__r1   r@   © r'   r%   r)   r)   a   s2   € € € € € ðð ð ðð ð ð ð r'   r)   c                 óº   — g }dd l }t          | d¬¦  «        5 } |j        |¦  «        }|D ]}|                     |¦  «         Œ	 d d d ¦  «         n# 1 swxY w Y   |S )Nr   r	   ©Únewline)ÚcsvÚopenÚ
DictReaderr:   )r-   ÚrowsrH   ÚcsvfileÚreaderÚrows         r%   Úload_results_from_csvrO   ~   s¶   € Ø€DØ€J€J€Jå	ˆh Ð	#Ñ	#Ô	#ð  wØ�” Ñ(Ô(ˆØð 	ð 	ˆCØ�KŠK˜ÑÔÐÐð	ðð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð ð €Ks   ˜+AÁAÁAc                 ó‚   — | D ].}|                      d¦  «        rt          | |         ¦  «        c S Œ/t          d¦  «        ‚)Nzaverage_latency(batch_size=z)Failed to get average_latency from output)Ú
startswithÚfloatÚRuntimeError)rN   Únames     r%   Úget_latencyrU   ‰   sT   € Øð $ð $ˆØ�?Š?Ð8Ñ9Ô9ð 	$Ý˜˜TœÑ#Ô#Ð#Ð#Ð#ð	$õ ÐBÑ
CÔ
CÐCr'   c                 ó–   — t          | ¦  «        }t          | d         ¦  «        }t          | d         ¦  «        }|dz  |dz  z
  |dz  z
  S )z@Scoring function based on 3 metrics. The larger score is better.Útop1_match_rateÚonnx_size_in_MBiè  é
   éd   )rU   rR   )rN   Úlatency_in_msrW   rX   s       r%   Úscorer\   ‘   sT   € å Ñ$Ô$€MÝ˜CÐ 1Ô2Ñ3Ô3€OÝ˜CÐ 1Ô2Ñ3Ô3€Oà˜TÑ! M°BÑ$6Ñ6¸È3Ñ9NÑNÐNr'   c                 ó¶  ‡
— t          ¦   «          t          d¦  «         i Š
|D ]}|‰
|d         <   Œt          t          |                      ¦   «         ˆ
fd„d¬¦  «        ¦  «        }t                               |› d|› �¦  «         t                               d|› d�¦  «         d	}d
}t          |                     ¦   «         ¦  «        D ]‘\  }\  }}	|	|k    r|}|	}|D ]|}|d         |k    rnt                               d                     ||	|t          |¦  «        t          |d         ¦  «        |d         |d         t          ¦   «         ¦  «        ¦  «          nŒ}Œ’d S )Nz
**********r/   c                 óJ   •— | d         t          ‰| d                  ¦  «        fS )Nr4   r   )r\   )ÚitemÚrow_maps    €r%   ú<lambda>zprint_wins.<locals>.<lambda>¥   s!   ø€ ˜d 1œg¥u¨W°T¸!´WÔ-=Ñ'>Ô'>Ð?€ r'   T)ÚkeyÚreversez Wins:z	Based on z* wins and a scoring function, the ranking:r   éÿÿÿÿza{:02d}: WINs={:02d}, run_id={}, latency={:5.2f}, top1_match={:.4f}, size={}_MB, experiment={}, {}rW   rX   Ú
experiment)ÚprintÚdictÚsortedÚitemsr<   ÚdebugÚinfoÚ	enumerateÚformatrU   rR   r   )ÚwinsrK   Ú	test_namerN   Úsorted_winsÚrankÚprevious_valueÚcountrb   Úvaluer`   s             @r%   Ú
print_winsru   š   sŸ  ø€ Ý	�G„G€GÝ	ˆ(�O„O€Oà€GØð %ð %ˆØ!$ˆ��H”ÑÐåÝØ�JŠJ‰LŒLØ?Ð?Ð?Ð?Øð	
ñ 	
ô 	
ñô €Kõ ‡L‚L�IÐ2Ð2 [Ð2Ð2Ñ3Ô3Ð3Ý
‡K‚KÐQ˜IÐQÐQÐQÑRÔRÐRà€DØ€NÝ(¨×):Ò):Ñ)<Ô)<Ñ=Ô=ð ð Ñˆ‰|��UØ�NÒ"Ð"ØˆDØˆàð 	ð 	ˆCØ�8Œ} Ò#Ð#Ý—’Øw×~Ò~ØØØÝ# CÑ(Ô(Ý˜cÐ"3Ô4Ñ5Ô5ØÐ-Ô.Ø˜LÔ)Ý5Ñ7Ô7ñ	ô 	ñô ð ð �ð $øðð r'   c                 ó\  — i }i }| D ]}|d         }d||<   d||<   Œt          |dd¬¦  «        5 }g d¢}t          j        ||¬¦  «        }|                     ¦   «          g d¢}	t	          | ¦  «        }
t          |
d	z
  ¦  «        D �]Q}| |         }t          |d
         t          ¦  «        rt          j	        |d
         ¦  «        }n|d
         }t          |d	z   |
d	¦  «        D �]ò}| |         }d}|	D ]}||         ||         k    rd} nŒ|sŒ+t          |d
         t          ¦  «        rt          j	        |d
         ¦  «        }n|d
         }	 t          j                             ||dd¬¦  «        \  }}n# t          $ r d}d}Y nw xY wt          j                             ||dd¬¦  «        \  }}|�_|dk     rYt          |d         ¦  «        t          |d         ¦  «        k    r||d         xx         d	z  cc<   n||d         xx         d	z  cc<   |dk     rYt          |d         ¦  «        t          |d         ¦  «        k    r||d         xx         d	z  cc<   n||d         xx         d	z  cc<   |d         |d         |d         t          |d         ¦  «        |d         |d         t          |d         ¦  «        ||||dœ}|                     |¦  «         �Œô�ŒS	 ddd¦  «         n# 1 swxY w Y   t"                               d|› �¦  «         t'          || d¦  «         t'          || d¦  «         dS )zRun U test and T test.r/   r   Úwr	   rF   )Ú
model_nameÚrun_id_1Úexperiment_1Útop1_match_rate_1Úrun_id_2Úexperiment_2Útop1_match_rate_2ÚU_statisticÚU_pvalueÚT_statisticÚT_pvalue)Ú
fieldnames)rx   r,   Úrunsr4   Útop1_match_rate_per_runTFz	two-sided)Úuse_continuityÚalternativeN)ÚaxisÚ	equal_vargš™™™™™©?rW   rx   re   z(U-Test and T-Test results are output to zU-TestzT-Test)rI   rH   Ú
DictWriterÚwriteheaderÚlenÚrangeÚ
isinstancer   ÚjsonÚloadsÚscipyÚstatsÚmannwhitneyuÚ
ValueErrorÚ	ttest_indrR   Úwriterowr<   rk   ru   )rK   Úoutput_csv_pathÚ
utest_winsÚ
ttest_winsrN   r/   rL   Úcolumn_namesÚwriterÚrequired_match_columnsÚnum_resultsÚiÚresult1ÚaÚjÚresult2Úall_matchedÚcolumnÚbÚutest_statisticÚutest_pvalueÚttest_statisticÚttest_pvalues                          r%   Úrun_significance_testrª   Ä   sJ  € à€JØ€JØð ð ˆØ�X”ˆØˆ
�6ÑØˆ
�6ÑÐå	ˆo˜s¨BÐ	/Ñ	/Ô	/ð O%°7ð
ð 
ð 
ˆõ ” °LÐAÑAÔAˆØ×ÒÑÔÐà!EÐ!EÐ!EÐÝ˜$‘i”iˆÝ�{ Q‘Ñ'Ô'ð ;	%ñ ;	%ˆAØ˜1”gˆGå˜'Ð";Ô<½cÑBÔBð 7Ý”J˜wÐ'@ÔAÑBÔB��àÐ5Ô6�å˜1˜q™5 +¨qÑ1Ô1ð 3%ñ 3%�Ø˜qœ'�à"�Ø4ð ð �FØ˜v”¨'°&¬/Ò9Ð9Ø&+˜Ø˜ð :ð #ð Øå˜gÐ&?Ô@Å#ÑFÔFð ;Ýœ
 7Ð+DÔ#EÑFÔF�A�AàÐ 9Ô:�Að(Ý49´K×4LÒ4LØ˜1¨T¸{ð 5Mñ 5ô 5Ñ1�O \ \øõ "ð (ð (ð (Ø&*�OØ#'�L�L�Lð(øøøõ 16´×0EÒ0EÀaÈÐQUÐaeÐ0EÑ0fÔ0fÑ-� àÐ+°¸tÒ0CÐ0CÝ˜WÐ%6Ô7Ñ8Ô8½5ÀÐIZÔA[Ñ;\Ô;\Ò\Ð\Ø" 7¨8Ô#4Ð5Ð5Ô5¸Ñ:Ð5Ð5Ñ5Ð5à" 7¨8Ô#4Ð5Ð5Ô5¸Ñ:Ð5Ð5Ñ5à $Ò&Ð&Ý˜WÐ%6Ô7Ñ8Ô8½5ÀÐIZÔA[Ñ;\Ô;\Ò\Ð\Ø" 7¨8Ô#4Ð5Ð5Ô5¸Ñ:Ð5Ð5Ñ5Ð5à" 7¨8Ô#4Ð5Ð5Ô5¸Ñ:Ð5Ð5Ñ5ð #*¨,Ô"7Ø '¨Ô 1Ø$+¨LÔ$9Ý).¨wÐ7HÔ/IÑ)JÔ)JØ '¨Ô 1Ø$+¨LÔ$9Ý).¨wÐ7HÔ/IÑ)JÔ)JØ#2Ø ,Ø#2Ø ,ðð �ð —’ Ñ$Ô$Ð$Ñ$ñg3%ð;	%ð)O%ð O%ð O%ñ O%ô O%ð O%ð O%ð O%ð O%ð O%ð O%øøøð O%ð O%ð O%ð O%õ` ‡K‚KÐL¸?ÐLÐLÑMÔMÐMÝˆz˜4 Ñ*Ô*Ð*Ýˆz˜4 Ñ*Ô*Ð*Ð*Ð*s7   ®DK"Å	&E0Å/K"Å0F	Å>K"Æ F	ÆEK"Ë"K&Ë)K&Úraw_onnx_modelc                 ó   — t          j        | ¦  «        }t          |¦  «        }|                     ¦   «         }|j        j        d         j        |v sJ ‚||j        j        d         j                 }|j        dk    r)t           	                    d|j        › �¦  «         |j        S t           
                    d|j        › d|j        › �¦  «         d S )Nr   ÚMatMulz#Found last MatMul node for logits: z-Failed to find MatMul node for logits. Found z	 of node )ÚonnxÚloadr   Úoutput_name_to_nodeÚgraphÚoutputrT   Úop_typer<   rk   Úwarning)r«   ÚmodelÚ
onnx_modelr°   Únodes        r%   Úget_last_matmul_node_namer¸   "  sÆ   € ÝŒI�nÑ%Ô%€EÝ˜5Ñ!Ô!€JØ$×8Ò8Ñ:Ô:ÐàŒ;Ô˜aÔ Ô%Ð)<Ð<Ð<Ð<Ð<Ø˜uœ{Ô1°!Ô4Ô9Ô:€DØ„|�xÒÐÝ�ŠÐE¸$¼)ÐEÐEÑFÔFÐFØŒyÐå
‡N‚NÐeÀ4Ä<ÐeÐeÐZ^ÔZcÐeÐeÑfÔfÐfØˆ4r'   c                 óÂ   — | j         }d|› d�                     ¦   «         }| j        r|                     d¦  «         |ddd|gz  }|r|                     dg|¢¦  «         |S )Nú-m ú -o --use_gpu -p fp16r   ú--io_block_listÚlogitsz--node_block_listú--op_block_list)Úmodel_name_or_pathÚsplitr   r:   Úextend)r$   Úlast_matmul_node_nameÚop_block_listrµ   Ú
parameterss        r%   Úget_mixed_precision_parametersrÅ   1  sŽ   € ØÔ#€EØ3�uÐ3Ð3Ð3×9Ò9Ñ;Ô;€JØÔ$ð 8Ø×ÒÐ6Ñ7Ô7Ð7ØØØØØð	ñ €Jð ð ?Ø×ÒÐ,Ð=¨}Ð=Ñ>Ô>Ð>àÐr'   ÚFastGeluÚLayerNormalizationÚtaskc                 óè   — t          |||¦  «        }d                     t          |¦  «        ¦  «        }|rd|› d�}nd|› d�}t          ¦   «         }|r	|d|› d�z   }|                      ||¦  «         d S )Nú,úMixed precision baseline + ú in FP32z=Mixed precision baseline (logits output and last MatMul node z	 in FP32)z (ú))rÅ   r   rh   r   r@   )rÈ   r$   rÂ   rÃ   rÄ   Úop_block_list_strrT   Úenv_varss           r%   Úrun_candidaterÐ   C  s    € õ 0°Ð6KÈ]Ñ[Ô[€JØŸš¥¨Ñ!6Ô!6Ñ7Ô7Ðàð pØHÐ->ÐHÐHÐHˆˆàoÐOdÐoÐoÐoˆå,Ñ.Ô.€HØð 'ØÐ&˜8Ð&Ð&Ð&Ñ&ˆà‡H‚HˆZ˜ÑÔÐÐÐr'   c                 ó   — | j         }d|› d�                     ¦   «         }| j        r|                     d¦  «         | j        r|                     d¦  «         d|› d�                     ¦   «         }| j        r|                     d¦  «         ||fS )Nrº   z -o -p fp32r   r   r»   )r¿   rÀ   r   r:   r   )r$   rµ   Úfp32_baselineÚfp16_baselines       r%   Úget_baselinesrÔ   X  s«   € ØÔ#€EØ,˜%Ð,Ð,Ð,×2Ò2Ñ4Ô4€MØ„|ð *Ø×Ò˜[Ñ)Ô)Ð)ØÔ$ð ;Ø×ÒÐ9Ñ:Ô:Ð:à6˜%Ð6Ð6Ð6×<Ò<Ñ>Ô>€MØÔ$ð ;Ø×ÒÐ9Ñ:Ô:Ð:à˜-Ð'Ð'r'   c                 óÄ  ‡— ddg}|                       ||z   d¦  «         dg}|                       ||z   d¦  «         |                       ||z   dgz   t          |¦  «        z   dgz   d¦  «         g }|}|D ]KŠdgˆfd	„|D ¦   «         z   }|                       ||z   |z   d
‰› d�¦  «        }	|	r|                     |	¦  «         ŒLt          |d„ ¬¦  «        }
t	          d|
¦  «         dS )z:Step 0 is to check which operator in FP16 causes most lossr¼   r½   zFP16 except logitsz--keep_io_typeszGraph I/O FP32, Other FP16r¾   z--force_fp16_initializerszFP32 except weights in FP16c                 ó    •— g | ]
}|‰k    ¯|‘ŒS rD   rD   )Ú.0ÚoÚops     €r%   ú
<listcomp>z$run_tuning_step0.<locals>.<listcomp>x  s   ø€ Ð.MÐ.MÐ.M°QÀQÈ"ÂWÀW¨qÀWÀWÀWr'   zFP32 except z in FP16c                 ó   — | d         S )NrW   rD   )Úys    r%   ra   z"run_tuning_step0.<locals>.<lambda>~  s   € ¸!Ð<MÔ:N€ r'   )rb   z<step 0: optimized operator causes the most loss in precisionN)r@   Úlistr:   Úminrf   )rÈ   rÓ   Úall_opsÚoptimized_opsÚfp32_logitsÚfp32_ioÚoptimized_ops_resultsÚop_listrÃ   r?   Ú
min_resultrÙ   s              @r%   Úrun_tuning_step0ræ   g  s>  ø€ à$ hÐ/€KØ‡H‚Hˆ]˜[Ñ(Ð*>Ñ?Ô?Ð?à Ð!€GØ‡H‚Hˆ]˜WÑ$Ð&BÑCÔCÐCð 	‡H‚HØ˜ÑÐ#4Ð"5Ñ5½¸W¹¼ÑEÐIdÐHeÑeØ%ñô ð ð
 ÐØ€GØð 1ð 1ˆØ*Ð+Ð.MÐ.MÐ.MÐ.M¸'Ð.MÑ.MÔ.MÑMˆØ—’˜-¨'Ñ1°MÑAÐC^ÐRTÐC^ÐC^ÐC^Ñ_Ô_ˆØð 	1Ø!×(Ò(¨Ñ0Ô0Ð0øõ Ð*Ð0NÐ0NÐOÑOÔO€JÝ	Ð
HÈ*ÑUÔUÐUÐUÐUr'   c                 óR   — |D ]#}d|g}|                       ||z   d|› d�¦  «         Œ$dS )zKStep 1 is to figure out which optimized operator in FP32 could benefit mostr¾   rË   rÌ   N)r@   )rÈ   Úmixed_precision_baselinerà   rÙ   rÃ   s        r%   Úrun_tuning_step1ré   ‚  sV   € àð 
ð 
ˆØ*¨BÐ/ˆØ�ŠØ$ }Ñ4Ø6¨"Ð6Ð6Ð6ñ	
ô 	
ð 	
ð 	
ð
ð 
r'   c           	      óÐ   ‡— g d¢}ˆfd„|D ¦   «         }‰D ]O}||vrIg |¢|‘}|                       g |¢d‘|¢d                     d                     |¦  «        |¦  «        ¦  «         ŒPdS )z�Assumed that you have run step 0 and 1 to figure out that Logits FP32 and some operators shall be in FP32,
    This step will try add one more operator.
    )rÆ   rÇ   ÚSkipLayerNormalizationc                 ó   •— g | ]}|‰v ¯|‘Œ	S rD   rD   )r×   Úxrà   s     €r%   rÚ   z$run_tuning_step2.<locals>.<listcomp>‘  s#   ø€ ÐDÐDÐD�a°°mÐ1CÐ1C�Ð1CÐ1CÐ1Cr'   r¾   z(Mixed precision baseline + {},{} in FP32rÊ   N)r@   rm   r   )rÈ   rè   rà   Úcandidate_fp32_opsÚfp32_opsrÙ   rÃ   s     `    r%   Úrun_tuning_step2rð   Œ  s¨   ø€ ð VÐUÐUÐØDÐDÐDÐDÐ-ÐDÑDÔD€HØð ð ˆØ�XÐÐØ+˜h˜O¨˜OˆMØ�HŠHØNÐ*ÐNÐ,=ÐNÀÐNØ:×AÒAÀ#Ç(Â(È8ÑBTÔBTÐVXÑYÔYñô ð øðð r'   c           	      óÌ  ‡
— t          j        d|j        |j        g ¬¦  «        }t	          |¦  «        \  }}|                      |d¦  «        }g }|r(d|v r$|d         r|d                              d¦  «        }nt          d¦  «        ‚g Š
|r(d|v r$|d         r|d                              d¦  «        Š
nt          d¦  «        ‚|j        st           
                    d	¦  «         d S |                      |d
¦  «         t          |d         ¦  «        }t          | ||g ¬¦  «         ˆ
fd„}|j        rGt          | |‰
|¦  «         t          ||g ¬¦  «        }	t!          | |	|¦  «         t#          | |	|¦  «         n2t          | || |g d¢¦  «        ¬¦  «         t          | ||dg¬¦  «         t          | || |g d¢¦  «        ¬¦  «         t          | || |g d¢¦  «        ¬¦  «         d S )NÚonnx_models)Ú
new_folderÚremove_existingzFP32 baselineÚoptimized_operatorsrÊ   z!Failed to get optimized operatorsÚ	operatorszFailed to get operatorsz5skip mixed precision since --use_gpu is not specifiedzFP16 baselineÚraw)rÃ   c                 ó    •— ˆfd„| D ¦   «         S )Nc                 ó   •— g | ]}|‰v ¯|‘Œ	S rD   rD   )r×   rÙ   rß   s     €r%   rÚ   z4run_parity.<locals>.get_fp32_ops.<locals>.<listcomp>À  s   ø€ Ð0Ð0Ð0�r "¨ - -� - - -r'   rD   )rí   rß   s    €r%   Úget_fp32_opsz run_parity.<locals>.get_fp32_ops¿  s   ø€ Ø0Ð0Ð0Ð0˜QÐ0Ñ0Ô0Ð0r'   )rë   rÇ   ÚAddrÆ   )rÆ   rë   rÇ   rû   )rÆ   ÚEmbedLayerNormalizationrë   rÇ   rû   )r   Úget_onnx_pathsr¿   r   rÔ   r@   rÀ   rS   r   r<   rk   r¸   rÐ   r   ræ   rÅ   ré   rð   )rÈ   r$   Úonnx_model_pathsrÒ   rÓ   r?   rà   rÂ   rú   rè   rß   s             @r%   Ú
run_parityrÿ   ›  s–  ø€ Ý!Ô0ØØÔØÔ0Øð	ñ ô Ðõ $1°Ñ#6Ô#6Ñ €M�=à�XŠX�m _Ñ5Ô5€Fà€MØð @Ð(¨FÐ2Ð2¸Ð?TÔ8UÐ2ØÐ4Ô5×;Ò;¸CÑ@Ô@ˆˆåÐ>Ñ?Ô?Ð?à€GØð 6�; &Ð(Ð(¨f°[Ô.AÐ(Ø˜Ô%×+Ò+¨CÑ0Ô0ˆˆåÐ4Ñ5Ô5Ð5ð Œ<ð Ý�ŠÐKÑLÔLÐLØˆà‡H‚Hˆ]˜OÑ,Ô,Ð,å5Ð6FÀuÔ6MÑNÔNÐõ �$˜Ð3À2ÐFÑFÔFÐFð1ð 1ð 1ð 1ð 1ð „xð UÝ˜˜}¨g°}ÑEÔEÐEÝ#AÀ$ÐH]ÐmoÐ#pÑ#pÔ#pÐ Ý˜Ð7¸ÑGÔGÐGÝ˜Ð7¸ÑGÔGÐGÐGåØØØ!Ø&˜,Ð'^Ð'^Ð'^Ñ_Ô_ð		
ñ 	
ô 	
ð 	
õ 	�d˜DÐ"7È
À|ÐTÑTÔTÐTõ ØØØØ"�lÐ#fÐ#fÐ#fÑgÔgð	ñ ô ð õ ØØØØ"�lØjÐjÐjñ
ô 
ð	ñ ô ð ð ð r'   Ú__main__rZ   é   i'  zŽNot enough test cases or runs to get stable results or test significance. Recommend test_cases >= 100, runs >= 20, test_cases * runs >= 10000.zOutput file zK existed. Please remove the file, or use either --skip_test or --overwrite.z6Remove existing file %s since --overwrite is specifiedzFailed to load csv z#Start running significance tests...z.csvz
.stats.csv)N):r   rH   r7   r�   ÚloggingÚosr®   Úscipy.statsr‘   Úbenchmark_helperr   r   Úconvert_to_onnxr   Úgpt2_helperr   r   r¶   r   Ú	getLoggerr<   r&   r)   rO   rU   r\   ru   rª   r   r¸   rÅ   rÐ   rÔ   ræ   ré   rð   rÿ   rA   r$   r   r,   r„   r´   ÚpathÚexistsr   r   rS   rk   ÚremoverÈ   r-   rK   r;   r=   r.   ÚreplaceÚsummary_csvrD   r'   r%   ú<module>r     sÙ  ðð €€€Ø 
€
€
€
Ø €€€Ø €€€Ø €€€Ø 	€	€	€	à €€€Ø Ð Ð Ð Ø HÐ HÐ HÐ HÐ HÐ HÐ HÐ HØ  Ð  Ð  Ð  Ð  Ð  Ø :Ð :Ð :Ð :Ð :Ð :Ð :Ð :Ø  Ð  Ð  Ð  Ð  Ð  à	ˆÔ	˜2Ñ	Ô	€ð@ð @ð @ð @ðFð ð ð ð ñ ô ð ð:ð ð ðDð Dð DðOð Oð Oð'ð 'ð 'ðT[+ð [+ð [+ð|¨cð ð ð ð ðð ð ð, Ð3Ð4ð	ð Ø
ðð ð ð ð*(ð (ð (ðVð Vð Vð6
ð 
ð 
ðð ð ðC�Zð Cð Cð Cð CðL ˆzÒÑØˆ?ÑÔ€DØ€L�”ÑÔÐà„˜ÒÐ ¤	¨B¢ °$´/ÀDÄIÑ2MÐPUÒ2UÐ2UØ�ŠðSñ	
ô 	
ð 	
ð
 
„w‡~‚~�d”hÑÔð  ¨¬ð  ØŒ~ð 	 Ø�,Øt˜tœxÐtÐtÐtñô ð ð �KŠKÐPÐRVÔRZÑ[Ô[Ð[ØˆBŒI�d”hÑÔÐàˆ:�d”o t¤y°$´(Ñ;Ô;€DàŒ>ð Øˆ
�4˜ÑÔÐðØ$Ð$ T¤]Ñ3Ô3ˆˆøØð ð ð Ø×ÒÐ>¨t¬}Ð>Ð>Ñ?Ô?Ð?ØŒ|ˆˆˆðøøøð ‡K‚KÐ5Ñ6Ô6Ð6Ø”-×'Ò'¨°Ñ=Ô=€KØÐ˜$ Ñ,Ô,Ð,Ð,Ð,ðA Ðs   ÆF# Æ#)GÇG