§
    û¸¸jÚX  ã                   ój  — d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZ	d dl
Z
d dlZd dlZd dlmZmZ d dlmZ d dlmZ d dlmZmZmZ d dlmZ d dlmZmZmZ d dlZ ej         e!¦  «        Z"de j#        fd	„Z$de j#        fd
„Z%d„ Z&d„ Z'd„ Z(d„ Z)d„ Z*d„ Z+d„ Z,d„ Z-e!dk    r e-¦   «          dS dS )é    N)Úmeasure_memoryÚsetup_logger)Úget_library_path)ÚORTModelForSpeechSeq2Seq)ÚProfilerActivityÚprofileÚrecord_function)Útrange)ÚAutoModelForSpeechSeq2SeqÚWhisperConfigÚWhisperProcessorÚargsc                 ó¦  ‡ ‡‡	‡
— ‰ j         dvrt          d¦  «        ‚ˆ fd„Šˆ fd„Š	‰ j        ‰ j        ‰ j        ‰ j        ‰ j        ‰ j        dœ}‰ j         dk    rÞ|                     ¦   «         D ]9\  }}t          j
        |gd|v rt          j        nt          j        ¬¦  «        ||<   Œ:‰ j        r)t          j
        ‰ j        gt          j        ¬¦  «        |d	<   ‰ j        r)t          j
        ‰ j        gt          j        ¬¦  «        |d
<   ‰ j        r)t          j
        ‰ j        gt          j        ¬¦  «        |d<   t&                               d‰ j        › �¦  «         ˆˆ	fd„}t-          ‰ |‰ j        ¦  «          |‰ j        ¦  «        }‰ j        r||d<   |S t&                               d¦  «         ‰ j         dk    rdndŠ
ˆ ˆ
fd„}t-          ‰ ||¦  «          ||¦  «        }‰ j         dk    r||d<   |S |                     ‰ j        rt4          j        nt4          j        ‰ j        ¬¦  «        |d<   ‰ j        |d<   d|d<   d|d<   ‰ j        r
‰ j        |d<   |S )N>   Úortúhf-ortúhf-pt-eagerúhf-pt-compilez/Unable to auto-detect inputs for provided modelc                  ób   •— t          j        ‰j        ¦  «        } t          j        | ¦  «        } | S ©N)ÚwhisperÚ
load_audioÚ
audio_pathÚpad_or_trim)Úaudior   s    €ú�/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/onnxruntime/transformers/models/whisper/benchmark.pyÚload_via_ffmpegz#get_inputs.<locals>.load_via_ffmpeg$   s*   ø€ ÝÔ" 4¤?Ñ3Ô3ˆÝÔ# EÑ*Ô*ˆØˆó    c                  ó
  •— t          ‰j        d¦  «        5 } t          j        t	          |                      ¦   «         ¦  «        t          j        ¬¦  «        }t          j        |g¦  «        }d d d ¦  «         n# 1 swxY w Y   |S )NÚrb©Údtype)Úopenr   ÚnpÚasarrayÚlistÚreadÚuint8Úarray)Úfr   r   s     €r   Úload_via_numpyz"get_inputs.<locals>.load_via_numpy)   s£   ø€ Ý�$”/ 4Ñ(Ô(ð 	&¨AÝ”J�t A§F¢F¡H¤H™~œ~µR´XÐ>Ñ>Ô>ˆEÝ”H˜e˜WÑ%Ô%ˆEð	&ð 	&ð 	&ñ 	&ô 	&ð 	&ð 	&ð 	&ð 	&ð 	&ð 	&øøøð 	&ð 	&ð 	&ð 	&ð ˆs   —AA8Á8A<Á?A<)Ú
max_lengthÚ
min_lengthÚ	num_beamsÚnum_return_sequencesÚlength_penaltyÚrepetition_penaltyr   Úpenaltyr    Údecoder_input_idsÚlogits_processorÚtemperaturezLoad audio: c                 ó0   •— | r
 ‰¦   «         n	 ‰¦   «         S r   © )Úonnx_e2er   r*   s    €€r   ú<lambda>zget_inputs.<locals>.<lambda>D   s!   ø€ ¸Ð%X ^ ^Ñ%5Ô%5Ð%5ÀÀÑGXÔGX€ r   Úaudio_streamzFeature extraction: r#   Úptc                 óT   •— ‰j                              | g‰‰j        ¬¦  «        j        S )N)Úreturn_tensorsÚsampling_rate)Ú	processorÚfeature_extractorr=   Úinput_features)r   r   Úreturn_types    €€r   r8   zget_inputs.<locals>.<lambda>P   s1   ø€  ¤×!AÒ!AØ	ˆ ¸4Ô;Mð "Bñ "ô "äð r   r@   )r!   ÚdeviceÚinputsÚno_repeat_ngram_sizeTÚearly_stoppingÚ	use_cacheÚforced_decoder_ids)Úbenchmark_typeÚ	Exceptionr+   r,   r-   r.   r/   r0   Úitemsr#   r(   Úfloat32Úint32Úhas_decoder_input_idsr2   Úhas_logits_processorr3   Úhas_temperaturer4   ÚloggerÚinfor   Útime_fnÚhas_audio_streamÚtoÚuse_fp16ÚtorchÚfloat16Útarget_devicerD   )r   rC   ÚkÚvÚload_audio_fnÚ
audio_dataÚprocessor_fnr@   r   r*   rA   s   `       @@@r   Ú
get_inputsr^       sÜ  øøøø€ ØÔÐ"SÐSÐSÝÐIÑJÔJÐJðð ð ð ð ð
ð ð ð ð ð ”oØ”oØ”^Ø $Ô 9ØÔ-Ø"Ô5ðð €Fð Ô˜eÒ#Ð#à—L’L‘N”Nð 	Xð 	X‰DˆAˆqÝœ ! ¸)Àq¸.¸.­B¬J¨JÍbÌhÐWÑWÔWˆF�1‰IˆIØÔ%ð 	]Ý*,¬(°DÔ4JÐ3KÕSUÔS[Ð*\Ñ*\Ô*\ˆFÐ&Ñ'ØÔ$ð 	[Ý)+¬°4Ô3HÐ2IÕQSÔQYÐ)ZÑ)ZÔ)ZˆFÐ%Ñ&ØÔð 	SÝ$&¤H¨dÔ.>Ð-?ÅrÄzÐ$RÑ$RÔ$RˆF�=Ñ!õ ‡K‚KÐ0˜tœÐ0Ð0Ñ1Ô1Ð1ØXÐXÐXÐXÐX€MÝˆD�- Ô!6Ñ7Ô7Ð7Ø�˜tÔ4Ñ5Ô5€JàÔð à!+ˆˆ~ÑØˆõ ‡K‚KÐ&Ñ'Ô'Ð'ØÔ-°Ò6Ð6�$�$¸D€Kðð ð ð ð €Lõ ˆD�, 
Ñ+Ô+Ð+Ø!�\ *Ñ-Ô-€NàÔ˜eÒ#Ð#à#1ˆÐÑ Øˆà%×(Ò(Ø#œ}Ð?�eŒmˆmµ%´-ÈÔHZð )ñ ô €Fˆ8Ñð &*Ô%>€FÐ!Ñ"Ø#€FÐÑØ€Fˆ;ÑàÔð >Ø'+Ô'=ˆÐ#Ñ$à€Mr   c                 ó@  — d\  }}d\  }}| j         dv r§| j        r| j        n| j        }t          j        ¦   «         }t	          j        || j        rt          j        nt          j	        d¬¦  «         
                    | j        ¦  «        }t          j        ¦   «         }| j         dk    rt          j        |¦  «        }n¥| j         dv r…t          j        ¦   «         }| j        |_        |                     t%          ¦   «         ¦  «         | j        r=d|_        d|_        | j        r(t          j        d¦  «         t          j        d¦  «         nt3          d	| j         › �¦  «        ‚| j         d
k    r�t5          | j        ¦  «        t8          u r| j        d         n| j        }t5          | j        ¦  «        t8          u r| j        d         nd }t          j        ¦   «         }t;          j        | j        |||d¬¦  «        }t          j        ¦   «         }| j         dk    rjt>                                d| j!        › �¦  «         t          j        ¦   «         }t          j"        | j!        || j        g¬¦  «        }t          j        ¦   «         }t>                                d||z
  › d�¦  «         |S )N)NN¾   r   r   T)Útorch_dtyperF   r   >   r   r   é   r   úCannot recognize r   )ÚproviderÚprovider_optionsÚsession_optionsÚuse_io_bindingr   zLoading model from )Ú	providerszLoaded model in ú s)#rH   Úhf_pt_model_pathÚ
model_nameÚtimer   Úfrom_pretrainedrU   rV   rW   rK   rT   rX   Úcompiler   ÚSessionOptionsr   Úenable_profilingÚregister_custom_ops_libraryr   ÚverboseÚlog_verbosity_levelÚlog_severity_levelÚtuneÚset_default_logger_severityÚset_default_logger_verbosityrI   ÚtypeÚexecution_providerÚtupler   Úhf_ort_dir_pathrP   rQ   Úort_model_pathÚInferenceSession)r   ÚmodelÚsess_optionsÚ
start_timeÚend_timeÚsourcerd   re   s           r   Ú	get_modelrƒ   h   s—  € Ø$Ñ€Eˆ<Ø%Ñ€J�ð ÔÐ>Ð>Ð>Ø*.Ô*?ÐT�Ô&Ð&ÀTÄ_ˆÝ”Y‘[”[ˆ
Ý)Ô9ØØ)-¬ÐI�œ˜½E¼MØð
ñ 
ô 
÷ Š"ˆTÔÑ
 Ô
 ð	 	õ
 ”9‘;”;ˆàÔ /Ò1Ð1Ý”M %Ñ(Ô(ˆEøà	Ô	Ð 1Ð	1Ð	1ÝÔ)Ñ+Ô+ˆØ(,¬ˆÔ%Ø×0Ò0Õ1AÑ1CÔ1CÑDÔDÐDØŒ<ð 	4Ø/0ˆLÔ,Ø./ˆLÔ+ØŒyð 4ÝÔ/°Ñ2Ô2Ð2ÝÔ0°Ñ3Ô3Ð3øõ ÐA¨DÔ,?ÐAÐAÑBÔBÐBàÔ˜hÒ&Ð&å15°dÔ6MÑ1NÔ1NÕRWÐ1WÐ1W�4Ô*¨1Ô-Ð-Ð]aÔ]tˆÝ9=¸dÔ>UÑ9VÔ9VÕZ_Ð9_Ð9_˜4Ô2°1Ô5Ð5ÐeiÐå”Y‘[”[ˆ
Ý(Ô8ØÔ ØØ-Ø(Øð
ñ 
ô 
ˆõ ”9‘;”;ˆàÔ˜eÒ#Ð#å�ŠÐ?¨$Ô*=Ð?Ð?Ñ@Ô@Ð@Ý”Y‘[”[ˆ
ÝÔ$ØÔØØÔ.Ð/ð
ñ 
ô 
ˆõ
 ”9‘;”;ˆå
‡K‚KÐ< 8¨jÑ#8Ð<Ð<Ð<Ñ=Ô=Ð=à€Lr   c                 ó.  — t          |¦  «        t          u r|d         n|}t          |¦  «        t          u r|d         n|}t          j        | j        ¦  «        }| j        dk    rt          | j        ¦  «        n t          | j        t          j
        d¬¦  «        }| j        r% ||¦  «        }t                               |¦  «         |D ]} ||¦  «         Œ| j        dk    rt          j                             |¦  «         t!          j        ¦   «         }	| j        dk    rt          | j        ¦  «        n t          | j        t          j
        d¬¦  «        }
|
D ]} ||¦  «         Œ| j        dk    rt          j                             |¦  «         t!          j        ¦   «         }| j        dk    rt                               d¦  «         d}||	z
  | j        z  }||z  }t                               d	|› d
�¦  «         t                               d|› d�¦  «         d S )Nr   rb   r   zWarm up)ÚfileÚdescÚcpuÚ	BenchmarkÚ z	Latency: ri   zThroughput: z qps)rx   rz   rV   rB   rX   rH   ÚrangeÚwarmup_runsr
   ÚsysÚstdoutrr   rP   rQ   ÚcudaÚsynchronizerl   Únum_runs)r   ÚfnrC   Úwarmup_inputsÚbenchmark_inputsÚtorch_deviceÚwarmup_rangeÚoutputsÚ_r€   Úbench_ranger�   Ú
batch_sizeÚlatencyÚ
throughputs                  r   rR   rR   «   s  € Ý!% f¡¤µÐ!6Ð!6�F˜1”I�I¸F€MÝ$(¨¡L¤LµEÐ$9Ð$9�v˜a”y�y¸vÐÝ”< Ô 2Ñ3Ô3€Lð
 Ô %Ò'Ð'õ 	ˆdÔÑÔÐå�DÔ$­3¬:¸IÐFÑFÔFð ð „|ð Ø�"�]Ñ#Ô#ˆÝ�Š�GÑÔÐàð ð ˆØ
ˆˆ=ÑÔÐÐð „{�eÒÐÝŒ
×Ò˜|Ñ,Ô,Ð,Ý”‘”€Jð Ô %Ò'Ð'õ 	ˆdŒmÑÔÐå�D”M­¬
¸ÐEÑEÔEð ð
 ð ð ˆØ
ˆÐÑÔÐÐà„{�eÒÐÝŒ
×Ò˜|Ñ,Ô,Ð,ÝŒy‰{Œ{€Hð Ô˜eÒ#Ð#Ý�Š�B‰Œˆà€JØ˜*Ñ$¨¬Ñ5€GØ˜gÑ%€Jå
‡K‚KÐ'˜GÐ'Ð'Ð'Ñ(Ô(Ð(Ý
‡K‚KÐ/˜zÐ/Ð/Ð/Ñ0Ô0Ð0Ø
€Fr   c                 ó,  — | j                              ¦   «         › d| j        › d| j        › d|j                             dd¦  «        › d|› dt          j                             ¦   «         d›�}d }| j         dv �rt          t          j
        t          j        gdd¬¦  «        5 }t          d¦  «        5   ||¦  «         d d d ¦  «         n# 1 swxY w Y   d d d ¦  «         n# 1 swxY w Y   |                     d¬	¦  «                             | j        | j        ¬
¦  «        }t"          j                             | j        |› d�¦  «        }t+          |d¦  «        5 }|                     |¦  «         d d d ¦  «         n# 1 swxY w Y   n ||¦  «         |› d�}|S )Nú-r—   z%Y-%m-%d_%H:%M:%Sr`   T)Ú
activitiesÚrecord_shapesÚprofile_memoryÚmodel_inferenceé   )Úgroup_by_stack_n)Úsort_byÚ	row_limitz.logÚwú.json)rH   ÚlowerÚ	precisionrB   Ú__name__ÚreplaceÚdatetimeÚnowr   r   ÚCPUÚCUDAr	   Úkey_averagesÚtableÚpt_filter_byÚpt_num_rowsÚosÚpathÚjoinÚ
log_folderr"   Úwrite)	r   r‘   rC   Úinputs_typeÚprefixÚfilenameÚprofÚ	prof_datar)   s	            r   Ú
profile_fnr¾   Ü   s›  € ð Ô#×)Ò)Ñ+Ô+ð  gð  g¨d¬nð  gð  g¸t¼{ð  gð  gÈRÌ[×M`ÒM`ÐadÐfiÑMjÔMjð  gð  gÐmxð  gð  gõ  |Dô  |M÷  |Qò  |Qñ  |Sô  |Sð  gð  gð  g€FØ€HàÔÐ>Ð>Ñ>åÝ(Ô,Õ.>Ô.CÐDÐTXÐimð
ñ 
ô 
ð 	àÝ Ð!2Ñ3Ô3ð ð Ø��6‘
”
�
ðð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð ð	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	ð
 ×%Ò%°qÐ%Ñ9Ô9×?Ò?ÈÔHYÐeiÔeuÐ?ÑvÔvˆ	å”7—<’< ¤°F°°°ÑAÔAˆÝ�(˜CÑ Ô ð 	 AØ�GŠG�IÑÔÐð	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	øð
 	ˆˆ6‰
Œ
ˆ
ð Ð#Ð#Ð#ˆà€OsH   ÂCÂ/CÂ;CÃC	ÃCÃC	ÃCÃC"Ã%C"ÅE8Å8E<Å?E<c                 óä  ‡‡— t          j        ¦   «         }t          j        |¦  «        }|                     d¬¦  «          ‰‰¦  «         t
                               d|                     d ¬¦  «        › d�¦  «         t          j        ¦   «          t          j
                             ¦   «          t          | j        dk    ˆˆfd„| j        ¬¦  «         t          j                             ¦   «          d S )Ngš™™™™™¹?)ÚintervalzCPU usage: ú%r‡   c                  ó   •—  ‰ ‰¦  «        S r   r6   )r‘   rC   s   €€r   r8   zmeasure_fn.<locals>.<lambda>  s   ø€ ¸r¸rÀ&¹z¼z€ r   )Úis_gpuÚfuncÚmonitor_type)r´   ÚgetpidÚpsutilÚProcessÚcpu_percentrP   rQ   ÚgcÚcollectrV   rŽ   Úempty_cacher   rB   rÅ   rŒ   r�   Úflush)r   r‘   rC   ÚpidÚprocesss    ``  r   Ú
measure_fnrÐ   ù   sÛ   øø€ å
Œ)‰+Œ+€CÝŒn˜SÑ!Ô!€GØ×Ò ÐÑ%Ô%Ð%à€B€v�J„J€JÝ
‡K‚KÐC˜g×1Ò1¸4Ð1Ñ@Ô@ÐCÐCÐCÑDÔDÐDõ „J�L„L€LÝ	„J×ÒÑÔÐÝ˜4œ;¨%Ò/Ð7IÐ7IÐ7IÐ7IÐ7IÐX\ÔXiÐjÑjÔjÐjõ „J×ÒÑÔÐÐÐr   c                 óª  ‡ ‡‡
— ˆfd„Š
ˆ ˆ
fd„}|}‰ j         dk    r ||¦  «         ‰ j        �rt          ‰ ||d¦  «        }‰ j         dk    �ræ|d t          d¦  «         …         }‰j        j                             ¦   «         }|dz   }t          j         	                    |¦  «        rXt                               d|› d	|› �¦  «         t          j        |t          j                             ‰ j        |¦  «        ¦  «         ‰j        j                             ¦   «         }|d
z   }t          j         	                    |¦  «        rXt                               d|› d	|› �¦  «         t          j        |t          j                             ‰ j        |¦  «        ¦  «         ‰j        j                             ¦   «         }|dz   }t          j         	                    |¦  «        rXt                               d|› d	|› �¦  «         t          j        |t          j                             ‰ j        |¦  «        ¦  «         d S t                               d¦  «         t%          ‰ ||¦  «          ||¦  «        \  }}	t                               dt          |d         ¦  «        › d�¦  «         t                               d|	d         › �¦  «         t'          ‰ ||¦  «         d S )Nc                 ó"   •—  ‰j         di | ¤Ž}|S )Nr6   )Úgenerate)rC   Úpredicted_idsr~   s     €r   Úget_pred_idsz&run_hf_inference.<locals>.get_pred_ids  s    ø€ à&˜œÐ0Ð0¨Ð0Ð0ˆØÐr   c                 ó¾   •—  ‰| ¦  «        }g }t          ‰j        ¦  «        D ]7}|                     ‰j                             |d¬¦  «        d         ¦  «         Œ8||fS )NT©Úskip_special_tokensr   )rŠ   r.   Úappendr>   Úbatch_decode)rC   rÔ   Útranscriptionr—   r   rÕ   s       €€r   Úgen_and_decz%run_hf_inference.<locals>.gen_and_dec  sq   ø€ à$˜ VÑ,Ô,ˆØˆÝ�tÔ0Ñ1Ô1ð 	jð 	jˆAØ× Ò  ¤×!<Ò!<¸]Ð`dÐ!<Ñ!eÔ!eÐfgÔ!hÑiÔiÐiÐiØ˜mÐ+Ð+r   r   zgen-and-decr   r§   z-encoder.jsonú	Renaming ú to z-decoder.jsonz-decoder-with-past.jsonz
Evaluating PyTorch...úGenerated token length: r   ú tokensúTranscription: )rH   r   r¾   ÚlenÚencoderÚsessionÚend_profilingr´   rµ   ÚisfilerP   ÚwarningÚrenamer¶   r·   ÚdecoderÚdecoder_with_pastrQ   rR   rÐ   )r   rC   r~   rÜ   Úgenerate_fnÚnew_lognameÚ
new_prefixÚold_lognamerÔ   rÛ   rÕ   s   ` `       @r   Úrun_hf_inferencerï     sÁ  øøø€ ðð ð ð ð ð
,ð ,ð ,ð ,ð ,ð ,ð  €KàÔ˜oÒ-Ð-àˆ�FÑÔÐà„|ñ Ý   {°F¸MÑJÔJˆØÔ (Ò*Ñ*à$ _­¨G©¬ } _Ô5ˆJàœ-Ô/×=Ò=Ñ?Ô?ˆKØ$ Ñ6ˆKÝŒw�~Š~˜kÑ*Ô*ð SÝ—’ÐI¨;ÐIÐI¸KÐIÐIÑJÔJÐJÝ”	˜+¥r¤w§|¢|°D´OÀ[Ñ'QÔ'QÑRÔRÐRàœ-Ô/×=Ò=Ñ?Ô?ˆKØ$ Ñ6ˆKÝŒw�~Š~˜kÑ*Ô*ð SÝ—’ÐI¨;ÐIÐI¸KÐIÐIÑJÔJÐJÝ”	˜+¥r¤w§|¢|°D´OÀ[Ñ'QÔ'QÑRÔRÐRàÔ1Ô9×GÒGÑIÔIˆKØ$Ð'@Ñ@ˆKÝŒw�~Š~˜kÑ*Ô*ð SÝ—’ÐI¨;ÐIÐI¸KÐIÐIÑJÔJÐJÝ”	˜+¥r¤w§|¢|°D´OÀ[Ñ'QÔ'QÑRÔRÐRàˆõ ‡K‚KÐ)Ñ*Ô*Ð*ÝˆD�+˜vÑ&Ô&Ð&Ø#. ;¨vÑ#6Ô#6Ñ €M�=Ý
‡K‚KÐI­3¨}¸QÔ/?Ñ+@Ô+@ÐIÐIÐIÑJÔJÐJÝ
‡K‚KÐ4 -°Ô"2Ð4Ð4Ñ5Ô5Ð5Ýˆt�[ &Ñ)Ô)Ð)Ð)Ð)r   c                 óÞ  ‡ ‡— dˆ ˆfd„	}ˆfd„}ˆfd„}ˆ fd„}‰ j         dk    r|n|} ||¦  «        }‰ j        r€t          ‰ ||d¦  «        }	‰                     ¦   «         }
t                               d|
› d	|	› �¦  «         t          j        |
t          j         	                    ‰ j
        |	¦  «        ¦  «         d S t                               d
¦  «         |}‰ j        r ||d¬¦  «        }||f}t          ‰ ||¦  «          ||¦  «        }‰ j         dk    r|                     ¦   «         }|d         }‰ j        r*t                               d|d         d         › �¦  «         nz ||d         d         ¦  «        }t                               dt!          |¦  «        › d�¦  «         ‰ j                             |d         d¬¦  «        d         }t'          |› ¦  «         t)          ‰ ||¦  «         d S )NFc                 óÀ  •— d„ ‰                      ¦   «         D ¦   «         }t          |                      ¦   «         ¦  «        }||z
  }t          |¦  «        r,t                               d|› �¦  «         t          d¦  «        ‚|r‰j        r| d         | d<   ||z
  }t          |¦  «        r&|D ]#}t                               d|› d�¦  «         | |= Œ$‰j	        dk    r„‰ 
                    ¦   «         }|                      ¦   «         D ]\  }}	|                     ||	¦  «         Œ‰                     ¦   «         D ])}
|                     |
j        ‰j	        ‰j        ¬	¦  «         Œ*|S | S )
Nc                 ó   — h | ]	}|j         ’Œ
S r6   ©Úname©Ú.0Úmodel_inputs     r   ú	<setcomp>z@run_ort_inference.<locals>.prepare_ort_inputs.<locals>.<setcomp>N  s   € ÐOÐOÐO¨[˜Ô(ÐOÐOÐOr   z(The following model inputs are missing: zEThere are missing inputs to the model. Please add them and try again.r+   r,   zRemoving unnecessary input 'z' from user provided inputsr‡   )Údevice_typeÚ	device_id)r^   ÚsetÚkeysrâ   rP   ÚerrorrI   ru   rQ   rB   Ú
io_bindingrJ   Úbind_cpu_inputÚget_outputsÚbind_outputrô   rú   )rC   ÚwarmupÚmodel_inputsÚuser_inputsÚmissing_inputsÚunnecessary_inputsÚunnecessary_inputrþ   rY   rZ   Úoutputr   r~   s              €€r   Úprepare_ort_inputsz-run_ort_inference.<locals>.prepare_ort_inputsL  s—  ø€ àOÐO¸E×<LÒ<LÑ<NÔ<NÐOÑOÔOˆÝ˜&Ÿ+š+™-œ-Ñ(Ô(ˆØ%¨Ñ3ˆÝˆ~ÑÔð 	eÝ�LŠLÐTÀNÐTÐTÑUÔUÐUÝÐcÑdÔdÐdàð 	8�d”ið 	8Ø#)¨,Ô#7ˆF�<Ñ ð )¨<Ñ7ÐÝÐ!Ñ"Ô"ð 	.Ø%7ð .ð .Ð!Ý—’ÐiÐ;LÐiÐiÐiÑjÔjÐjØÐ,Ð-Ð-ð Œ;˜%ÒÐØ×)Ò)Ñ+Ô+ˆJØŸš™œð 0ð 0‘��1Ø×)Ò)¨!¨QÑ/Ô/Ð/Ð/Ø×+Ò+Ñ-Ô-ð gð g�Ø×&Ò& v¤{ÀÄÐW[ÔWeÐ&ÑfÔfÐfÐfØÐàˆr   c                 ó2   •— ‰                      | ¦  «         | S r   )Úrun_with_iobinding)rþ   r~   s    €r   Úwith_io_bindingz*run_ort_inference.<locals>.with_io_bindingj  s   ø€ à× Ò  Ñ,Ô,Ð,ØÐr   c                 ó4   •— ‰                      d | ¦  «        }|S r   )Úrun)rC   r–   r~   s     €r   Úwithout_io_bindingz-run_ort_inference.<locals>.without_io_bindingo  s   ø€ à—)’)˜D &Ñ)Ô)ˆØˆr   c                 ó†   •— ‰j         | v r6t          j        | ‰j         k    ¦  «        d         d         }| d |dz   …         S | S )Nr   rb   )Úeos_token_idr#   Úwhere)r  Ú	first_endr   s     €r   Úhandle_outputz(run_ort_inference.<locals>.handle_outputt  sJ   ø€ ØÔ Ð&Ð&Ýœ ¨4Ô+<Ò!<Ñ=Ô=¸aÔ@ÀÔCˆIØ˜/˜I¨™M˜/Ô*Ð*àˆr   r‡   Úe2erÝ   rÞ   z
Evaluating ONNX Runtime...T)r  r   rá   rß   rà   r×   )F)rB   r   r¾   rå   rP   rç   r´   rè   rµ   r¶   r·   rQ   ru   rR   Úcopy_outputs_to_cpurS   râ   r>   rÚ   ÚprintrÐ   )r   rC   r~   r	  r  r  r  rë   Ú
ort_inputsrì   rî   Úort_evaluate_inputsÚort_warmup_inputsÚort_outputsÚactual_outputrÛ   s   ` `             r   Úrun_ort_inferencer  K  se  øø€ ðð ð ð ð ð ð ð<ð ð ð ð ð
ð ð ð ð ð
ð ð ð ð ð &*¤[°EÒ%9Ð%9�/�/Ð?Q€KØ#Ð# FÑ+Ô+€Jà„|ð Ý   {°JÀÑFÔFˆð ×)Ò)Ñ+Ô+ˆÝ�ŠÐA ;ÐAÐA°KÐAÐAÑBÔBÐBÝ
Œ	�+�rœwŸ|š|¨D¬O¸[ÑIÔIÑJÔJÐJàˆõ ‡K‚KÐ.Ñ/Ô/Ð/Ø$ÐØ„yð >Ø.Ð.¨v¸dÐCÑCÔCÐØ0°*Ð=ÐåˆD�+Ð2Ñ3Ô3Ð3Ø�+˜jÑ)Ô)€KØ„{�eÒÐØ!×5Ò5Ñ7Ô7ˆØ˜a”.€KàÔð 	"å�ŠÐ9 k°!¤n°QÔ&7Ð9Ð9Ñ:Ô:Ð:Ð:ð &˜ k°!¤n°QÔ&7Ñ8Ô8ˆÝ�ŠÐJ­s°=Ñ/AÔ/AÐJÐJÐJÑKÔKÐKØœ×3Ò3°KÀ´NÐX\Ð3Ñ]Ô]Ð^_Ô`ˆå�Ð Ñ!Ô!Ð!åˆt�[ *Ñ-Ô-Ð-Ð-Ð-r   c                 ó¤   — | j         dv rt          | ||¦  «         d S | j         dk    rt          | ||¦  «         d S t          d| j         › �¦  «        ‚)N>   r   r   r   r   rc   )rH   rï   r  rI   )r   rC   r~   s      r   Úrun_inferencer  £  si   € ØÔÐHÐHÐHÝ˜˜v uÑ-Ô-Ð-Ð-Ð-Ø	Ô	 Ò	%Ð	%Ý˜$ ¨Ñ.Ô.Ð.Ð.Ð.åÐA¨DÔ,?ÐAÐAÑBÔBÐBr   c            	      óº	  — t          j        ¦   «         } |                      ddt          dg d¢¬¦  «         |                      ddt          dd¬	¦  «         |                      d
dt          ddg d¢d¬¦  «         |                      dt          dd¬¦  «         |                      dt          dd¬¦  «         |                      dt          dd¬¦  «         |                      ddt          dd¬	¦  «         |                      ddt          t          j                             ¦   «         rdndg d¢¬ ¦  «         |                      d!d"t          d#¬$¦  «         |                      d%d&t          d'¬$¦  «         |                      d(d)t          d*¬$¦  «         |                      d+t          d,¬$¦  «         |                      d-t          d.d/¬¦  «         |                      d0t          d1¬$¦  «         |                      d2t          d#¬$¦  «         |                      d3t          d4¬$¦  «         |                      d5t          d4¬$¦  «         |                      d6t          d7¬$¦  «         |                      d8t          d7¬$¦  «         |                      d9t          d:¬$¦  «         |                      d;t          d<d=¬¦  «         |                      d>t          d4d?¬¦  «         |                      d@t          d7dA¬¦  «         |                      dBdCdD¬E¦  «         |                      dFt          dGdH¬¦  «         |                      dIt          dJdK¬¦  «         |                      dLdCdD¬E¦  «         |                      dMt          t          j
                             dN¦  «        dO¬¦  «         |                      dPdCdDdQ¬R¦  «         |                      ¦   «         }t          j                             |j        ¦  «         t	          j        |j        ¦  «         |j        |_        dS|j        v rt|j                             ¦   «         › dT�|_        |j        dUk    r|j        dV|j        if|_        n2|j        dWk    r'|j        |j        d4|j        rd4nd#dXœf|_        d|_        |j        dYk    r|j        s
J dZ¦   «         ‚|j        dSk    r|j        s
J d[¦   «         ‚t5          j        |j        ¦  «        |_        |S )\Nz-btz--benchmark-typeT)r   r   r   r   )rx   ÚrequiredÚchoicesz-mz--model-namez;Hugging Face name of model (e.g. 'openai/whisper-large-v2'))rx   r!  Úhelpz-pz--precisionÚfp32)Úint8Úfp16r$  zePrecision for model. For ONNX models, the model's precision should be set before running this script.)rx   r!  Údefaultr"  r#  z--hf-pt-model-pathr‰   zNPath to directory containing all PyTorch files (e.g. tokenizer, PyTorch model))rx   r'  r#  z--hf-ort-dir-pathzaPath to directory containing all ONNX files (e.g. tokenizer, encoder, decoder, decoder_with_past)z--ort-model-pathzPath to ONNX modelz-az--audio-pathz%Path to audio file for E2E evaluationz-dz--devicerŽ   r‡   )r‡   rŽ   Úrocm)rx   r'  r"  z-idz--device-idr   )rx   r'  z-wz--warmup-runsr¢   z-nz
--num-runsé
   z--seedé   z--sampling-ratei€>  zSampling rate for audio (in Hz)z--max-lengthiÀ  z--min-lengthz--num-beamsrb   z--num-return-sequencesz--length-penaltyg      ð?z--repetition-penaltyz--no-repeat-ngram-sizeé   z--decoder-input-idsz[]zìThe forced decoder ids for generation. Format is [start token, timestamp token, language token, task token]. Default is [start token]. See `decoder_input_ids` in https://github.com/microsoft/Olive/tree/main/examples/whisper for details.z--logits-processorzLWhether to use timestamps logits processor or not (0 for false, 1 for true).z--temperaturez!Temperature value for generation.z	--profileFÚ
store_true)r'  Úactionz--pt-filter-byÚself_cpu_time_totalz"What to filter PyTorch profiler byz--pt-num-rowsiè  z.Number of rows for PyTorch profiler to displayz	--verbosez--log-folderú.zFolder to cache log filesz--tunezFOnly used by ROCm EP, enable TunableOp tuning to select fastest kernel)r'  r-  r#  r   ÚExecutionProviderÚCUDAExecutionProviderrú   ÚROCMExecutionProvider)rú   Útunable_op_enableÚtunable_op_tuning_enabler   z,Please specify a path to `--hf-ort-dir-path`z+Please specify a path to `--ort-model-path`)ÚargparseÚArgumentParserÚadd_argumentÚstrrV   rŽ   Úis_availableÚintÚfloatr´   rµ   r¶   Ú
parse_argsr#   ÚrandomÚseedÚmanual_seedrB   rÅ   rH   Úupperry   rú   ru   r{   r|   ÚastÚliteral_evalr2   )Úparserr   s     r   r<  r<  ¬  s˜  € ÝÔ$Ñ&Ô&€Fà
×ÒØØÝØØAÐAÐAð ñ ô ð ð ×ÒØØÝØØJð ñ ô ð ð ×ÒØØÝØØØ(Ð(Ð(Øtð ñ ô ð ð ×ÒØÝØØ]ð	 ñ ô ð ð ×ÒØÝØØpð	 ñ ô ð ð ×ÒØÝØØ!ð	 ñ ô ð ð ×Ò˜˜nµ3ÀÐLsÐÑtÔtÐtØ
×ÒØØÝÝœ*×1Ò1Ñ3Ô3Ð>��¸Ø'Ð'Ð'ð ñ ô ð ð ×Ò˜˜}µ3ÀÐÑBÔBÐBØ
×Ò˜˜oµCÀÐÑCÔCÐCØ
×Ò˜˜lµ¸bÐÑAÔAÐAØ
×Ò˜¥s°AÐÑ6Ô6Ð6ð ×ÒÐ)µ¸UÐIjÐÑkÔkÐkð ×Ò˜­S¸#ÐÑ>Ô>Ð>Ø
×Ò˜­S¸!ÐÑ<Ô<Ð<Ø
×Ò˜­C¸ÐÑ;Ô;Ð;Ø
×ÒÐ0µsÀAÐÑFÔFÐFØ
×ÒÐ*µÀÐÑDÔDÐDØ
×ÒÐ.µUÀCÐÑHÔHÐHØ
×ÒÐ0µsÀAÐÑFÔFÐFð ×ÒØÝØð |ð	 ñ ô ð ð ×ÒØÝØØ[ð	 ñ ô ð ð ×ÒØÝØØ0ð	 ñ ô ð ð ×Ò˜¨U¸<ÐÑHÔHÐHØ
×ÒØ�sÐ,AÐHlð ñ ô ð ð ×Ò˜­c¸4ÐFvÐÑwÔwÐwØ
×Ò˜¨U¸<ÐÑHÔHÐHØ
×Ò˜­S½"¼'¿,º,ÀsÑ:KÔ:KÐRmÐÑnÔnÐnØ
×ÒØØØØUð	 ñ ô ð ð ×ÒÑÔ€Dõ „I‡N‚N�4”9ÑÔÐÝ	Ô�d”iÑ Ô Ð àœ€DÔà�Ô#Ð#Ð#Ø%)¤[×%6Ò%6Ñ%8Ô%8Ð"KÐ"KÐ"KˆÔØÔ"Ð&=Ò=Ð=Ø'+Ô'>ÀÈdÌnÐ@]Ð&^ˆDÔ#Ð#ØÔ$Ð(?Ò?Ð?àÔ'à!%¤Ø)*Ø59´YÐ0E°°ÀAðð ð'ˆDÔ#ð !ˆDŒKð Ô˜hÒ&Ð&ØÔ#ÐSÐSÐ%SÑSÔSÐ#ØÔ˜eÒ#Ð#ØÔ"ÐQÐQÐ$QÑQÔQÐ"õ !Ô-¨dÔ.DÑEÔE€DÔà€Kr   c                  óÂ  — t          ¦   «         } t          | j        ¦  «         t                               | j        ¦  «         dt          j        j        _	        t          j        | j        ¦  «        }t          j        | j        ¦  «        }| j        dk    r
d| j        › �n| j        }| j        dk    }t#          | d|¦  «         t#          | d|¦  «         t#          | d|¦  «         t#          | dd	¦  «         t#          | d
|j        ¦  «         t                               d| j        › �¦  «         t)          | ¦  «        }| j        dk    rxd„ |                     ¦   «         D ¦   «         }d|v | _        t#          | dd|v ¦  «         t#          | dd|v ¦  «         t#          | dd|v ¦  «         | j        g k    r|j        g| _        t-          | ¦  «        }t3          | ||¦  «         d S )NTr‡   zcuda:r&  r>   rX   rU   rS   Fr  zForced decoder prompt ids: r   c                 ó   — h | ]	}|j         ’Œ
S r6   ró   rõ   s     r   rø   zmain.<locals>.<setcomp>T  s   € ÐSÐSÐS°˜KÔ,ÐSÐSÐSr   r9   rM   r2   rN   r3   rO   r4   )r<  r   rr   rP   rQ   Ú__dict__rV   ÚbackendsÚcudnnÚ	benchmarkr   rm   rk   r   rB   rú   r©   Úsetattrr  r2   rƒ   rH   r^   rS   Údecoder_start_token_idr  )r   Úconfigr>   rX   rU   r~   Úort_model_inputsrC   s           r   ÚmainrN  =  sâ  € Ý‰<Œ<€DÝ�”ÑÔÐÝ
‡K‚K�”ÑÔÐØ%)…E„NÔÔ"åÔ*¨4¬?Ñ;Ô;€FÝ Ô0°´ÑAÔA€IØ04´¸uÒ0DÐ0DÐ,˜DœNÐ,Ð,Ð,È$Ì+€MØŒ~ Ò'€HåˆD�+˜yÑ)Ô)Ð)ÝˆD�/ =Ñ1Ô1Ð1ÝˆD�*˜hÑ'Ô'Ð'ÝˆDÐ$ eÑ,Ô,Ð,ÝˆD�. &Ô"5Ñ6Ô6Ð6å
‡K‚KÐF¨dÔ.DÐFÐFÑGÔGÐGõ �d‰OŒO€EØÔ˜eÒ#Ð#àSÐSÀ×@PÒ@PÑ@RÔ@RÐSÑSÔSÐØ .Ð2BÐ BˆÔÝ�Ð-Ð/BÐFVÐ/VÑWÔWÐWÝ�Ð,Ð.@ÐDTÐ.TÑUÔUÐUÝ�Ð'¨Ð:JÐ)JÑKÔKÐKàÔ! RÒ'Ð'Ø&,Ô&CÐ%DˆDÔ"å˜ÑÔ€FÝ�$˜ Ñ&Ô&Ð&Ð&Ð&r   Ú__main__).r5  rA  r¬   rÊ   Úloggingr´   rŒ   rl   Únumpyr#   rÇ   rV   r   Úbenchmark_helperr   r   Úonnxruntime_extensionsr   Úoptimum.onnxruntimer   Útorch.profilerr   r   r	   Útqdmr
   Útransformersr   r   r   Úonnxruntimer   Ú	getLoggerrª   rP   Ú	Namespacer^   rƒ   rR   r¾   rÐ   rï   r  r  r<  rN  r6   r   r   ú<module>r[     s  ðð €€€Ø 
€
€
€
Ø €€€Ø 	€	€	€	Ø €€€Ø 	€	€	€	Ø 
€
€
€
Ø €€€à Ð Ð Ð Ø €€€Ø €€€Ø €€€Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø 8Ð 8Ð 8Ð 8Ð 8Ð 8Ø EÐ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ Ð Ð Ð Ð Ð Ø SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ Sà Ð Ð Ð à	ˆÔ	˜8Ñ	$Ô	$€ðE�XÔ'ð Eð Eð Eð EðP@�HÔ&ð @ð @ð @ð @ðF.ð .ð .ðbð ð ð:ð ð ð$=*ð =*ð =*ð@U.ð U.ð U.ðpCð Cð CðNð Nð Nðb!'ð !'ð !'ðH ˆzÒÐØ€D�F„F€F€F€Fð Ðr   