§
    û¸¸j�<  ã                   ó.  — d dl Z d dlZd dlZd dlZd dlmZ d dlZd dlZd dlmZm	Z	m
Z
mZmZ d dlmZmZmZmZ d dlmZ d dlmZ d dlmZ d dlmZ  ej        d	¦  «        Zdd
„Zd„ Zedk    r' e¦   «         Z eej         ¦  «          ee¦  «         dS dS )é    N)Údatetime)Ú	PrecisionÚcreate_onnxruntime_sessionÚget_ort_environment_variablesÚprepare_environmentÚsetup_logger)ÚDEFAULT_TOLERANCEÚMODEL_CLASSESÚPRETRAINED_GPT2_MODELSÚ
Gpt2Helper)Úversion)ÚQuantizeHelper)Ú
AutoConfig)Ú__version__Ú c                 óz  — t          j        ¦   «         }|                     dddt          dd                     t
          ¦  «        z   ¬¦  «         |                     ddt          d	t          t          j        ¦   «         ¦  «        d
d                     t          j        ¦   «         ¦  «        z   ¬¦  «         |                     ddt          t          j
                             dd¦  «        d¬¦  «         |                     ddt          t          j
                             dd¦  «        d¬¦  «         |                     dddt          d¬¦  «         |                     ddddd¬¦  «         |                     ddddd¬¦  «         |                     d¬ ¦  «         |                     d!t          d"dg d#¢d$¬%¦  «         |                     d&ddd'¬¦  «         |                     d¬(¦  «         |                     d)d*t          t          j        t          t          ¦  «        d+¬,¦  «         |                     d-ddd.¬¦  «         |                     d¬/¦  «         |                     d0d1d2t          d3gd4¬5¦  «         |                     d6d2t          d3gd7¬5¦  «         |                     d8d9d2t          g d:¢d;¬5¦  «         |                     d<d=dd d>¬?¦  «         |                     d@dt          dAdB¬¦  «         |                     dCdd¬D¦  «         |                     d¬E¦  «         |                     dFdd¬D¦  «         |                     d¬G¦  «         |                     dHdd¬D¦  «         |                     d¬I¦  «         |                     dJdd¬D¦  «         |                     d¬K¦  «         |                     | ¦  «        }|S )LNz-mz--model_name_or_pathTz;Model path, or pretrained model name selected in the list: z, )ÚrequiredÚtypeÚhelpz--model_classFÚGPT2LMHeadModelz!Model type selected in the list: )r   r   ÚdefaultÚchoicesr   z--cache_dirú.Úcache_modelsz%Directory to cache pre-trained models)r   r   r   r   z
--onnx_dirÚonnx_modelszDirectory to store onnx modelsz--test_timeséd   z8Number of repeat times to get average inference latency.)r   r   r   r   z-vz--validate_onnxÚ
store_truezValidate ONNX model)r   Úactionr   z-oz--optimize_onnxz'Use optimizer.py to optimize onnx model)Úoptimize_onnxz--stager   )r   é   é   a6  Stage in generation: 1 (initial decoder), 2 (decoder), 0 (both). 1 - decode the first token when past_sequence_length is zero; 2 - decode the remaining tokens when past_sequence_length is not zero; 0 - one onnx model for both stages 1 and 2. Note that we will optimize 1 and 2 differently for best performance.)r   r   r   r   r   z	--use_gpuzuse GPU for inference)Úuse_gpuz-pz--precisionzfPrecision of model to run. fp32 for full precision, fp16 for half precision, and int8 for quantization)r   r   r   r   z--torchscriptzuse Torchscript)Útorchscriptz-bz--batch_sizesú+r    z
batch size)Únargsr   r   r   z--sequence_lengthsz!sequence lengths (excluding past)z-sz--past_sequence_lengths)é   é   é    é@   é€   é   zpast sequence lengthsz-rz--result_csvz$CSV file for saving summary results.)r   r   r   z--thread_numéÿÿÿÿzThreads to usez--include_copy_output_latency)r   r   )Úinclude_copy_output_latencyz	--verbose)Úverbosez--output_torch_latency)Úoutput_torch_latencyz--disable_io_binding)Údisable_io_binding)ÚargparseÚArgumentParserÚadd_argumentÚstrÚjoinr   Úlistr
   ÚkeysÚosÚpathÚintÚset_defaultsr   ÚFLOAT32Ú
parse_args)ÚargvÚparserÚargss      úƒ/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/onnxruntime/transformers/models/gpt2/benchmark_gpt2.pyÚparse_argumentsrB   !   sb  € ÝÔ$Ñ&Ô&€Fà
×ÒØØØÝØJÈTÏYÊYÕWmÑMnÔMnÑnð ñ ô ð ð ×ÒØØÝØ!Ý•]Ô'Ñ)Ô)Ñ*Ô*Ø0°4·9²9½]Ô=OÑ=QÔ=QÑ3RÔ3RÑRð ñ ô ð ð ×ÒØØÝÝ”—’˜S .Ñ1Ô1Ø4ð ñ ô ð ð ×ÒØØÝÝ”—’˜S -Ñ0Ô0Ø-ð ñ ô ð ð ×ÒØØØÝØGð ñ ô ð ð ×ÒØØØØØ"ð ñ ô ð ð ×ÒØØØØØ6ð ñ ô ð ð ×Ò eÐÑ,Ô,Ð,à
×ÒØÝØØØ�	�	ðOð ñ ô ð ð ×Ò˜¨e¸LÐOfÐÑgÔgÐgØ
×Ò ÐÑ&Ô&Ð&à
×ÒØØÝÝÔ!Ý•Y‘”Øuð ñ ô ð ð ×Ò˜°%ÀÐSdÐÑeÔeÐeØ
×Ò EÐÑ*Ô*Ð*à
×Ò˜˜o°S½sÈQÈCÐVbÐÑcÔcÐcà
×ÒØØÝØ�Ø0ð ñ ô ð ð ×ÒØØ!ØÝØ)Ð)Ð)Ø$ð ñ ô ð ð ×ÒØØØØØ3ð ñ ô ð ð ×Ò˜°½SÈ"ÐScÐÑdÔdÐdà
×ÒÐ7À%ÐP\ÐÑ]Ô]Ð]Ø
×Ò°EÐÑ:Ô:Ð:à
×Ò˜¨e¸LÐÑIÔIÐIØ
×Ò ÐÑ&Ô&Ð&à
×ÒÐ0¸5ÈÐÑVÔVÐVØ
×Ò¨UÐÑ3Ô3Ð3à
×ÒÐ.¸À|ÐÑTÔTÐTØ
×Ò¨5ÐÑ1Ô1Ð1à×Ò˜TÑ"Ô"€Dà€Kó    c                 ó„  — t          j        t          ¦  «        t          j        d¦  «        k     rt          d¦  «        ‚t                               d| › �¦  «         | j        t          j        k    r| j	        r| j
        s
J d¦   «         ‚| j        t          j        k    r| j
        r
J d¦   «         ‚| j        dk    r| j        dgk    s
J d¦   «         ‚t          j        | j        dk    rt#          j        d	¬
¦  «        n| j        ¦  «         t'          t          j                             ¦   «         ¦  «         | j        }| j        }t1          ||| j
        ¦  «         t2          | j                 d         }t6          }t9          j        | j        | j        |¬¦  «        }|                     | j        ||¬¦  «        }t          j         | j
        rdnd¦  «        }| !                    |¦  «         |j"        dk    }| #                    || j        | j        d	|¬¦  «        }	|	d         }
t2          | j                 d         }| $                    |||
| j%        |||¬¦  «         | j	        s| j        t          j&        k    �r|	| j        t          j        k    rtO          | j        ¦  «        nd         }
| 	                    |	d         |
| j        t          j        k    |j(        j)        |j(        j*        |d	| j        ¬¦  «         | j        t          j        k    rlt                               d¦  «         tW          j,        |
|	d         |¦  «         tW          j-        |¦  «        }t                               d¦  «         |	d         }
| j        r|                     |||||¬¦  «        }t]          |
| j
        d| j        | j%        ¬¦  «        }|€d S | /                    ta          | j1        ¦  «        ta          | j        ¦  «        ta          | j2        ¦  «        || j        ¦  «        }| 3                    ||| j        t          j        k    ¦  «        }| j4        p8d 5                    tm          j7        ¦   «          8                    d¦  «        ¦  «        }ts          |dd¬¦  «        5 }g d ¢}tu          j;        ||¬!¦  «        }| <                    ¦   «          | j1        D �]F}| j2        D �]:}| j        D �].}|dk    r|dk    r|dk    sJ ‚t           =                    d"|||¦  «         | >                    ||||j)        |j*        |j"        |j?        || j        t          j        k    ||¬#¦  «        }| /                    ||||| j        ¦  «        }	 | j@        s| jA        r«| B                    ||| jC        ¦  «        \  }}t‰          |¦  «        D ]{\  }}t‹          |tŒ          ¦  «        r<t           =                    d$|› d%t�          |¦  «        › d&|d         jH        › �¦  «         ŒVt           =                    d$|› d'|jH        › �¦  «         Œ|nd }d }| jI        r | J                    ||| jC        ¦  «        \  }}n)| K                    ||||| jC        d| jL        ¬(¦  «        \  }}| j@        r¶|}| jI        s@g }|D ];}| M                    | N                    ¦   «          O                    ¦   «         ¦  «         Œ<| P                    ||| j        t¢          | j                 t¢          | j                 ¬)¦  «        r.t                               d*t¢          | j                 › d+�¦  «         t                               d,||||| jI        rd-nd|rd.nd¦  «         | j        | j        | j        t¥          ¦   «         | j
        | j        | j	        | j        |||| jI        |r|d/›nd0|d/›d œ} | S                    | ¦  «         �Œõ# t¨          $ r. t           U                    d1d	¬2¦  «         Y    d d d ¦  «         d S w xY w�Œ<�ŒH	 d d d ¦  «         n# 1 swxY w Y   t                               d3|› �¦  «         |S )4Nz3.1.0z/This tool requires transformers 3.1.0 or later.z
Arguments:z'fp16 requires --optimize_onnx --use_gpuzquantization only supports CPUr    r   z<past_sequence_lengths shall be 0 for stage==1 (init decoder)T)Úlogical)r#   Ú	cache_dir)ÚconfigrF   zcuda:0Úcpué   )Úhas_pastÚ
new_folderÚrawr!   )Úhas_position_idsÚhas_attention_maskÚfp32)Úauto_mixed_precisionÚstagezquantizing model...Úint8zfinished quantizing modelF)Úenable_all_optimizationÚnum_threadsr.   zbenchmark_result_{}.csvz%Y%m%d-%H%M%SÚar   )ÚmodeÚnewline)Ú
model_nameÚmodel_classrQ   Úenvironment_variablesÚgpuÚ	precisionÚ	optimizerr#   Ú
batch_sizeÚsequence_lengthÚpast_sequence_lengthr0   Útorch_latencyÚonnxruntime_latency)Ú
fieldnameszMRunning test for batch_size=%d sequence_length=%d past_sequence_length=%d ...)Úfloat16rM   rN   ztorch output z is tuple of size z, shape z shape )Úreturn_numpyr-   )rY   ÚrtolÚatolz:Pytorch and ONNX Runtime outputs are all close (tolerance=z).zZbatch_size=%d, sequence_length=%d, past_sequence_length=%d, onnxruntime_latency=%.2f %s %sz(disable_io_binding)z, torch_latency={torch_latency}z.2fÚNoneÚ	Exception)Úexc_infozResults are saved to file )Vr   ÚparseÚtransformers_versionÚRuntimeErrorÚloggerÚinfor\   r   ÚFLOAT16r   r"   ÚINT8rQ   Úpast_sequence_lengthsÚtorchÚset_num_threadsÚ
thread_numÚpsutilÚ	cpu_countÚprintÚ
__config__Úparallel_inforF   Úonnx_dirr   r
   rY   r   r   Úfrom_pretrainedÚmodel_name_or_pathr#   ÚdeviceÚtoÚn_layerÚget_onnx_pathsÚexport_onnxr.   r<   r4   rG   Únum_attention_headsÚhidden_sizer   Úquantize_onnx_modelÚquantize_torch_modelr   Úget_output_shapesÚmaxÚbatch_sizesÚsequence_lengthsÚget_output_buffersÚ
result_csvÚformatr   ÚnowÚstrftimeÚopenÚcsvÚ
DictWriterÚwriteheaderÚdebugÚget_dummy_inputsÚ
vocab_sizeÚvalidate_onnxr/   Úpytorch_inferenceÚ
test_timesÚ	enumerateÚ
isinstanceÚtupleÚlenÚshaper0   Úonnxruntime_inferenceÚ$onnxruntime_inference_with_binded_ior-   ÚappendrH   ÚnumpyÚcompare_outputsr	   r   Úwriterowri   Úerror)!r@   rF   Ú
output_dirrY   Ú
gpt2helperrG   Úmodelr~   Úuse_external_data_formatÚonnx_model_pathsÚonnx_model_pathÚuse_paddingÚsessionÚmax_output_shapesÚoutput_buffersÚcsv_filenameÚcsv_fileÚcolumn_namesÚ
csv_writerr^   r_   r`   Údummy_inputsÚoutput_shapesÚoutputsra   ÚiÚvalueÚort_outputsÚort_latencyÚcopy_outputsÚoutputÚrows!                                    rA   Úmainr¾   ¨   sÄ	  € Ý„}Õ)Ñ*Ô*­W¬]Øñ.ô .ò ð õ ÐLÑMÔMÐMå
‡K‚KÐ#˜TÐ#Ð#Ñ$Ô$Ð$Ø„~�Ô*Ò*Ð*ØÔ!Ð] d¤lÐ]Ð]Ð4]Ñ]Ô]Ð2à„~�œÒ'Ð'Ø”<ÐAÐAÐ!AÑAÔAÐà„z�Q‚€ØÔ)¨a¨SÒ0Ð0Ð0Ð2pÑ0Ô0Ð0å	Ô¸D¼OÈqÒ<PÐ<P�&Ô*°4Ð8Ñ8Ô8Ð8ÐVZÔVeÑfÔfÐfÝ	�%Ô
×
(Ò
(Ñ
*Ô
*Ñ+Ô+Ð+à”€IØ”€JÝ˜	 :¨t¬|Ñ<Ô<Ð<å Ô 0Ô1°!Ô4€KÝ€JÝÔ'¨Ô(?ÈTÔM]ÐirÐsÑsÔs€FØ×'Ò'¨Ô(?ÈÐZcÐ'ÑdÔd€Eõ Œ\ d¤lÐ=˜(˜(¸Ñ>Ô>€FØ	‡H‚HˆVÑÔÐØ%œ~°Ò2ÐØ!×0Ò0ØØÔØÔØØ+ð 1ñ ô Ðð ' uÔ-€OÝ Ô 0Ô1°!Ô4€KØ×ÒØØØØŒØ Ø$Ø&ð ñ ô ð ð Ôð 7˜Tœ^­yÔ/@Ò@Ñ@Ø*À$Ä.ÕT]ÔTbÒBbÐBb­3¨t¬~Ñ+>Ô+>Ð+>ÐhnÔoˆØ× Ò Ø˜UÔ#ØØŒN�iÔ/Ò/ØŒLÔ,ØŒLÔ$Ø$Ø!%Ø”*ð 	!ñ 		
ô 		
ð 		
ð Œ>�Yœ^Ò+Ð+Ý�KŠKÐ-Ñ.Ô.Ð.ÝÔ.¨Ð@PÐQWÔ@XÐZrÑsÔsÐsÝ"Ô7¸Ñ>Ô>ˆEÝ�KŠKÐ3Ñ4Ô4Ð4Ø.¨vÔ6ˆOàÔð 
Ø×&Ò&ØØØØ(Ø*ð 'ñ 
ô 
ˆõ )ØØŒØ %Ø”OØ”ðñ ô €Gð €Øˆð #×4Ò4ÝˆDÔÑÔÝˆDÔ&Ñ'Ô'ÝˆDÔ!Ñ"Ô"ØØÔñô Ðð  ×2Ò2Ð3DÀfÈdÌnÕ`iÔ`qÒNqÑrÔr€Nà”?ÐpÐ&?×&FÒ&FÅxÄ|Á~Ä~×G^ÒG^Ð_nÑGoÔGoÑ&pÔ&p€LÝ	ˆl ¨bÐ	1Ñ	1Ô	1ð C$°Xð
ð 
ð 
ˆõ  ”^ H¸ÐFÑFÔFˆ
Ø×ÒÑ Ô Ð àÔ*ð o	$ñ o	$ˆJØ#'Ô#8ð n$ñ n$�Ø,0Ô,Fð m$ñ m$Ð(Ø%¨š>˜>¨oÀÒ.AÐ.AÐFZÐ^_ÒF_ÐF_ÐF_Ð_Ý—L’LØgØ"Ø'Ø,ñ	ô ð ð $.×#>Ò#>Ø"Ø,Ø'ØÔ2ØÔ*ØœØÔ)ØØ!%¤µ9Ô3DÒ!DØ)4Ø+6ð $?ñ $ô $�Lð %/×$@Ò$@Ø"Ø,Ø'ØØÔ(ñ%ô %�MðO$ØÔ-ð 1°Ô1Jð 1Ø5?×5QÒ5QÐRWÐYeÐgkÔgvÑ5wÔ5wÑ2˜G ]õ -6°gÑ,>Ô,>ð Zð Z¡  5Ý#-¨eµUÑ#;Ô#;ð !ZÝ$*§L¢LØ(q¸Ð(qÐ(qÍSÐQVÉZÌZÐ(qÐ(qÐafÐghÔaiÔaoÐ(qÐ(qñ%&ô %&ð %&ð %&õ %+§L¢LÐ1XÀÐ1XÐ1XÈ5Ì;Ð1XÐ1XÑ$YÔ$YÐ$YÐ$YðZð '+˜GØ,0˜MàÔ2ð Ø7A×7WÒ7WØ '¨°t´ñ8ô 8Ñ4˜K¨¨ð 8B×7fÒ7fØ 'Ø ,Ø .Ø -Ø $¤Ø-2Ø<@Ô<\ð 8gñ 8ô 8Ñ4˜K¨ð  Ô-ð "Ø+6˜LØ#'Ô#:ð Nà/1 Ø.9ð !Nð !N FØ$0×$7Ò$7¸¿
º
¹¼×8JÒ8JÑ8LÔ8LÑ$MÔ$MÐ$MÐ$Mà)×9Ò9Ø 'Ø ,Ø,0Ô,<Ý%6°t´~Ô%FÝ%6°t´~Ô%Fð  :ñ  ô  ð 	"õ !'§¢ð %GÕarÐswô  tBô  bCð  %Gð  %Gð  %Gñ!"ô !"ð !"õ ŸšØxØ&Ø+Ø0Ø'Ø6:Ô6MÐUÐ2Ð2ÐSUØANÐVÐ=Ð=ÐTVñô ð ð +/Ô*AØ+/Ô+;Ø%)¤ZÝ5RÑ5TÔ5TØ#'¤<Ø)-¬Ø)-Ô);Ø+/Ô+;Ø*4Ø/>Ø4HØ26Ô2IØGTÐ-`°Ð-CÐ-CÐ-CÐZ`Ø6AÐ3GÐ3Gðð ˜ð  #×+Ò+¨CÑ0Ô0Ð0Ñ0øÝ$ð $ð $ð $ÝŸš [¸4˜Ñ@Ô@Ð@Ø#˜t˜t˜tðGC$ð C$ð C$ñ C$ô C$ð C$ð C$ð C$ðB$øøøñWm$ñn$ðo	$ð)C$ð C$ð C$ñ C$ô C$ð C$ð C$ð C$ð C$ð C$ð C$øøøð C$ð C$ð C$ð C$õJ ‡K‚KÐ;¨\Ð;Ð;Ñ<Ô<Ð<ØÐs8   ÒC#`Õ1I_ß
`ß&`ß2`à`à`à`à`Ú__main__)N)!r1   r‘   Úloggingr8   r   rv   rs   Úbenchmark_helperr   r   r   r   r   Úgpt2_helperr	   r
   r   r   Ú	packagingr   Úquantize_helperr   Útransformersr   r   rl   Ú	getLoggerrn   rB   r¾   Ú__name__r@   r.   © rC   rA   ú<module>rÉ      s‡  ðð €€€Ø 
€
€
€
Ø €€€Ø 	€	€	€	Ø Ð Ð Ð Ð Ð à €€€Ø €€€ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ]Ð \Ð \Ð \Ð \Ð \Ð \Ð \Ð \Ð \Ð \Ð \Ø Ð Ð Ð Ð Ð Ø *Ð *Ð *Ð *Ð *Ð *Ø #Ð #Ð #Ð #Ð #Ð #Ø <Ð <Ð <Ð <Ð <Ð <à	ˆÔ	˜2Ñ	Ô	€ðDð Dð Dð DðNoð oð oðd ˆzÒÐØˆ?ÑÔ€DØ€L�”ÑÔÐØ€Dˆ�J„J€J€J€Jð ÐrC   