§
    û¸¸j%a  ã                  ó  — d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlmZ d dlmZmZ d dlmZmZmZmZ d dlZ ej        e¦  «        Zdd„Zd	„ Z d
„ Z!d„ Z"d„ Z#d„ Z$d„ Z%edk    r e%¦   «          dS dS )é    )ÚannotationsN)Úsetup_logger)Úadd_io_bindings_as_tensorsÚget_initial_inputs_and_outputs)Ú
AutoConfigÚAutoModelForCausalLMÚAutoTokenizerÚBitsAndBytesConfigÚargsúargparse.Namespacec                ó   — | j         dv �r«d }| j        dk    rx| j        dk    rmt          dddt          j        ¬¦  «        }t          j        | j        dk    r| j        n| j	        | j
        | j        | j        | j        dd|| j        d	i¬
¦	  «	        }nò	 t          j        | j        dk    r| j        n| j	        | j
        | j        | j        | j        d| j        dk    rdnd¬¦  «                             | j        ¦  «        }n…# t"          $ rx}t%          d|¦  «         t          j        | j        dk    r| j        n| j	        | j
        | j        | j        | j        dd¬¦  «                             | j        ¦  «        }Y d }~nd }~ww xY w|                     ¦   «          | j         dk    rt	          j        |¦  «        }nHt+          j        ¦   «         }| j        dk    rdd| j        ifnd}t+          j        | j        ||g¬¦  «        }|S )N¾   úpt-eagerú
pt-compileÚint4ÚcudaTÚnf4)Úload_in_4bitÚbnb_4bit_use_double_quantÚbnb_4bit_quant_typeÚbnb_4bit_compute_dtypeÚ Úflash_attention_2Ú80GB)Ú	cache_dirÚtorch_dtypeÚuse_auth_tokenÚtrust_remote_codeÚ	use_cacheÚattn_implementationÚquantization_configÚ
max_memoryÚsdpa)r   r   r   r   r   r    z&Try to load a model using eager mode: Úeagerr   ÚCUDAExecutionProviderÚ	device_idÚCPUExecutionProvider)Úsess_optionsÚ	providers)Úbenchmark_typeÚonnx_precisionÚdevicer
   ÚtorchÚfloat16r   Úfrom_pretrainedÚhf_dir_pathÚ
model_namer   r   ÚauthÚtrustr&   ÚtoÚtarget_deviceÚ	ExceptionÚprintÚevalÚcompileÚortÚSessionOptionsÚInferenceSessionÚonnx_model_path)r   ÚmodelÚ
bnb_configÚer(   Úeps         úƒ/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/onnxruntime/transformers/models/llama/benchmark_e2e.pyÚ	get_modelrC   8   sV  € ØÔÐ8Ð8Ñ8ØˆØÔ &Ò(Ð(¨T¬[¸FÒ-BÐ-BÝ+Ø!Ø*.Ø$)Ý',¤}ð	ñ ô ˆJõ )Ô8Ø$(Ô$4¸Ò$:Ð$:�Ô Ð ÀÄØœ.Ø Ô,Ø#œyØ"&¤*ØØ$7Ø$.Ø œN¨FÐ3ð
ñ 
ô 
ˆEˆEð)Ý,Ô<Ø(,Ô(8¸BÒ(>Ð(>�DÔ$Ð$ÀDÄOØ"œnØ $Ô 0Ø#'¤9Ø&*¤jØ"Ø@DÄÈvÒ@UÐ@UÐ)<Ð)<Ð[aðñ ô ÷ ’"�TÔ'Ñ(Ô(ð �øõ ð )ð )ð )õ Ð>ÀÑBÔBÐBÝ,Ô<Ø(,Ô(8¸BÒ(>Ð(>�DÔ$Ð$ÀDÄOØ"œnØ $Ô 0Ø#'¤9Ø&*¤jØ"Ø(/ðñ ô ÷ ’"�TÔ'Ñ(Ô(ð �����øøøøð	)øøøð 	�
Š
‰ŒˆàÔ ,Ò.Ð.Ý”M %Ñ(Ô(ˆEøõ Ô)Ñ+Ô+ˆð Œ{˜fÒ$Ð$ð % {°D´NÐ&CÐDÐDà'ð 	õ
 Ô$ TÔ%9ÈÐacÐ`dÐeÑeÔeˆà€Ls   ÂA+C= Ã=
E?ÄA.E:Å:E?c                ó  — | j         dk    r3t          j        ¦   «         5   |di |¤Ž}d d d ¦  «         n# 1 swxY w Y   d }| j         dv r0| j        dk    r$t          j                             | j        ¦  «         n1t          |||| j        | j	        ¦  «        }| 
                    ¦   «          t          j        ¦   «         }t          |¦  «        D ]—}| j         dv rct          j        ¦   «         5   |di |¤Ž}| j        dk    r$t          j                             | j        ¦  «         d d d ¦  «         n# 1 swxY w Y   Œn|                     |¦  «         |                     ¦   «          Œ˜t          j        ¦   «         }||z
  |z  }	|	|fS )Nr   r   Úcpu© )r*   r-   Úno_gradr,   r   Úsynchronizer5   r   Úuse_fp16Úuse_buffer_shareÚsynchronize_inputsÚtimeÚperf_counterÚrangeÚrun_with_iobindingÚsynchronize_outputs)
r   r>   ÚrunsÚinputsÚoutputsÚ
io_bindingÚstartÚ_ÚendÚavgs
             rB   Úrun_inferencerY   x   s
  € ØÔ˜lÒ*Ð*ÝŒ]‰_Œ_ð 	&ð 	&Ø�e�o�o˜f�o�oˆGð	&ð 	&ð 	&ñ 	&ô 	&ð 	&ð 	&ð 	&ð 	&ð 	&ð 	&øøøð 	&ð 	&ð 	&ð 	&ð €JØÔÐ8Ð8Ð8ØŒ;˜%ÒÐÝŒJ×"Ò" 4Ô#5Ñ6Ô6Ð6øå/°°v¸wÈÌÐW[ÔWlÑmÔmˆ
Ø×%Ò%Ñ'Ô'Ð'õ ÔÑÔ€EÝ�4‰[Œ[ð -ð -ˆØÔÐ"<Ð<Ð<Ý”‘”ð ?ð ?Ø˜%˜/˜/ &˜/˜/�Ø”; %Ò'Ð'Ý”J×*Ò*¨4Ô+=Ñ>Ô>Ð>ð?ð ?ð ?ñ ?ô ?ð ?ð ?ð ?ð ?ð ?ð ?øøøð ?ð ?ð ?ð ?øð
 ×$Ò$ ZÑ0Ô0Ð0Ø×*Ò*Ñ,Ô,Ð,Ð,å
Ô
Ñ
Ô
€CØ�‰;˜$Ñ
€CØ�ˆ<Ðs   Ÿ	4´8»8Ã+8D/Ä/D3	Ä6D3	c           
     ó¶   — t          ¦   «          t          ||||| j        | j        | j        | j        ¦  «        \  }}t          | || j        ||¦  «        \  }}||fS ©N)Úclear_cacher   r5   rI   rJ   ÚenginerY   Úwarmup_runs)	r   r>   ÚconfigÚ	tokenizerÚprompt_lengthÚpromptrR   rS   rV   s	            rB   Úprepare_model_for_inferencerc   —   sd   € Ý�M„M€MÝ4Ø�	˜=¨&°$Ô2DÀdÄmÐUYÔUjÐlpÔlwñô �O€FˆGõ ˜t U¨DÔ,<¸fÀgÑNÔN�J€A€wØ�7ˆ?Ðó    c                 óh   — t          j        ¦   «          t          j                             ¦   «          d S r[   )ÚgcÚcollectr-   r   Úempty_cacherF   rd   rB   r\   r\       s'   € Ý„J�L„L€LÝ	„J×ÒÑÔÐÐÐrd   c                óä   — t          j        | ddddddddd	|d
z  › d�d|d
z  › d�d	|› d�d|› d�ddg¬¦  «        }|                     |d¬¦  «         t                               d|› d�¦  «         d S )Nz
Batch SizezPrompt LengthzPrompt Processing Latency (ms)z"Prompt Processing Throughput (tps)zSampling Latency (ms)zSampling Throughput (tps)z"First Token Generated Latency (ms)z&First Token Generated Throughput (tps)úAverage Latency of First é   z Tokens Generated (ms)úAverage Throughput of First z Tokens Generated (tps)zWall-Clock Latency (s)zWall-Clock Throughput (tps))ÚcolumnsF)ÚindexzResults saved in ú!)ÚpdÚ	DataFrameÚto_csvÚloggerÚinfo)ÚresultsÚfilenameÚ
gen_lengthÚdfs       rB   Úsave_resultsry   ¥   sº   € Ý	ŒØàØØ,Ø0Ø#Ø'Ø0Ø4ØO¨
°a©ÐOÐOÐOØS¨:¸©?ÐSÐSÐSØJ¨
ÐJÐJÐJØN¨:ÐNÐNÐNØ$Ø)ð
ð
ñ 
ô 
€Bð( ‡I‚Iˆh˜e€IÑ$Ô$Ð$Ý
‡K‚KÐ/ HÐ/Ð/Ð/Ñ0Ô0Ð0Ð0Ð0rd   c                 óþ  — t          j        ¦   «         } |                      ddt          dg d¢¬¦  «         |                      ddt          dd	¬
¦  «         |                      ddddd¬¦  «         |                      ddddd¬¦  «         |                      ddt          t          j                             dd¦  «        d¬¦  «         |                      dt          dd¬¦  «         |                      dddd¬¦  «         |                      d d!dt          j                             dd"d#d$¦  «        d%¬&¦  «         |                      d'ddd(¬¦  «         |                      d)ddd*¬¦  «        f |                      d+d,d-¬.¦  «         |                      d/d0d1¬.¦  «         |                      d2d3dt          d4g d5¢d6¬7¦  «         |                      d8d9t          d:d;¬¦  «         |                      d<d=t          t          j	         
                    ¦   «         rd>nd?d?d>g¬@¦  «         |                      dAdBt          dC¬D¦  «         |                      dEdFt          dG¬D¦  «         |                      dHdIt          dJ¬D¦  «         |                      dKt          dL¬D¦  «         |                      ¦   «         }t          j                             |j        ¦  «         t          j        |j        ¦  «         dM|j        v rKt#          |dN|j                             ¦   «         › dO�¦  «         |j        dPk    r|j        dQ|j        if|_        |j        dMk    r|j        s
J dR¦   «         ‚|j                             dS¦  «        |_        |j                             dS¦  «        |_        t#          |dT|j        ¦  «         |j        dUv s|j        dVk    r|j        d?k    rd4ndW|_        |j        d?k    r
dX|j        › �n|j        }|j        dWk    rt          j        nt          j        }|j        dMk    rdMndY}t#          |dZ|¦  «         t#          |d[|¦  «         t#          |d\|¦  «         t#          |d]|j        dWk    ¦  «         |j        o|dMk    |_        |S )^Nz-btz--benchmark-typeT)r   r   r:   )ÚtypeÚrequiredÚchoicesz-mz--model-nameFz<Hugging Face name of model (e.g. 'meta-llama/Llama-2-7b-hf'))r{   r|   Úhelpz-az--authÚ
store_truez5Use Hugging Face authentication token to access model)ÚdefaultÚactionr~   z-tz--trustzeWhether or not to allow for custom models defined on the Hugging Face Hub in their own modeling filesz-cz--cache-dirú.Úmodel_cachez´Path to directory containing all Hugging Face files (e.g. config, tokenizer, PyTorch model). Use when loading model as `AutoModel.from_pretrained(model_name, cache_dir=cache_dir)`.)r{   r€   r~   z--hf-dir-pathr   z Path to directory containing all Hugging Face files (e.g. config, tokenizer, PyTorch model). Use when loading model as `AutoModel.from_pretrained(folder_path)`.z-oz--onnx-model-pathzPath to ONNX model)r|   r~   z-fz--prompts-fileÚmodelsÚllamazprompts.jsonzsJSON file containing entries in the format 'prompt length: prompt' where prompt length = tokenized length of prompt)r|   r€   r~   z--use_buffer_sharez3Use when GroupQueryAttention (GQA) is in ONNX modelz--anomaly-filteringzõUse this flag to filter anomaly accelerator times for tokens generated.               This may give more accurate latency and throughput metrics for tokens generated.               Wall-clock metrics are still reported with anomaly times though.z-bz--batch-sizesz1 2)r€   z-sz--prompt-lengthsz16 64 256 1024z-pz--precisionÚfp32)r   Úint8Úfp16r†   zePrecision for model. For ONNX models, the model's precision should be set before running this script.)r|   r{   r€   r}   r~   z-gz--generation-lengthé   z Number of new tokens to generatez-dz--devicer   rE   )r{   r€   r}   z-idz--device-idr   )r{   r€   z-wz--warmup-runsé   z-nz
--num-runséd   z--seedrk   r:   Úexecution_providerÚExecutionProviderr%   r&   z,Please specify a path to `--onnx-model-path`ú r+   >   r†   r‡   r   rˆ   zcuda:Úptr5   r   r]   rI   )ÚargparseÚArgumentParserÚadd_argumentÚstrÚosÚpathÚjoinÚintr-   r   Úis_availableÚ
parse_argsÚnpÚrandomÚseedÚmanual_seedr*   Úsetattrr,   ÚupperrŒ   r&   r=   Úbatch_sizesÚsplitÚprompt_lengthsÚ	precisionr.   Úfloat32rJ   )Úparserr   r5   r   r]   s        rB   Úget_argsr¦   ¾   s~  € ÝÔ$Ñ&Ô&€Fà
×ÒØØÝØØ1Ð1Ð1ð ñ ô ð ð ×ÒØØÝØØKð ñ ô ð ð ×ÒØØØØØDð ñ ô ð ð ×ÒØØØØØtð ñ ô ð ð ×ÒØØÝÝ”—’˜S -Ñ0Ô0ð Dð ñ ô ð ð ×ÒØÝØð pð	 ñ ô ð ð ×ÒØØØØ!ð	 ñ ô ð ð ×ÒØØØÝ”—’˜S (¨G°^ÑDÔDð Cð ñ ô ð ð ×ÒØØØØBð	 ñ ô ð ð 	×ÒØ!ØØðPð	 	ñ 	
ô 	
ð	ð 	ð ×ÒØØØð ñ ô ð ð ×ÒØØØ ð ñ ô ð ð ×ÒØØØÝØØ0Ð0Ð0Øtð ñ ô ð ð ×ÒØØÝØØ/ð ñ ô ð ð ×ÒØØÝÝœ*×1Ò1Ñ3Ô3Ð>��¸Ø˜�ð ñ ô ð ð ×Ò˜˜}µ3ÀÐÑBÔBÐBØ
×Ò˜˜oµCÀÐÑCÔCÐCØ
×Ò˜˜lµ¸cÐÑBÔBÐBØ
×Ò˜¥s°AÐÑ6Ô6Ð6à×ÒÑÔ€Dõ „I‡N‚N�4”9ÑÔÐÝ	Ô�d”iÑ Ô Ð ð �Ô#Ð#Ð#Ý�Ð*¨t¬{×/@Ò/@Ñ/BÔ/BÐ,UÐ,UÐ,UÑVÔVÐVØÔ"Ð&=Ò=Ð=Ø'+Ô'>ÀÈdÌnÐ@]Ð&^ˆDÔ#ð Ô˜eÒ#Ð#ØÔ#ÐSÐSÐ%SÑSÔSÐ#àÔ'×-Ò-¨cÑ2Ô2€DÔØÔ-×3Ò3°CÑ8Ô8€DÔõ ˆDÐ" D¤NÑ3Ô3Ð3à”.Ð$4Ð4Ð4¸¼È6Ò9QÐ9QÐVZÔVaÐejÒVjÐVjˆˆÐqwð 	„Nð 15´¸uÒ0DÐ0DÐ,˜DœNÐ,Ð,Ð,È$Ì+€MØ#'¤>°VÒ#;Ð#;•%”-�-ÅÄ€KØÔ)¨UÒ2Ð2ˆUˆU¸€FÝˆD�/ =Ñ1Ô1Ð1ÝˆD�- Ñ-Ô-Ð-ÝˆD�(˜FÑ#Ô#Ð#ÝˆD�*˜dœn°Ò6Ñ7Ô7Ð7à Ô1ÐE°fÀ²o€DÔà€Krd   c                 ób  ‡9‡:— t          ¦   «         } t          d¦  «         t                               | j        ¦  «         d }t          | j        ¦  «        5 }t          j        |d„ ¬¦  «        }d d d ¦  «         n# 1 swxY w Y   t          j
        | j        dk    r| j        n| j        | j        | j        | j        ¬¦  «        }t!          j
        | j        dk    r| j        n| j        | j        | j        | j        ¬¦  «        }t#          | ¦  «        }g }t%          j        | j        | j        ¦  «        D �	]<\  }}t-          |¦  «        t-          |¦  «        }}t                               d|› d|› �¦  «         t/          ¦   «          || j        z   }	||vrOt3          t5          j        d|› d	| j        › d
| j        › d|› d|› d|› d|› d|› d| j        › d|› d�¦  «        ¦  «        ‚||         g|z  }
||g}	 t                               d¦  «         t9          | |||||
¦  «        \  }}t;          | || j        ||¦  «        \  }}|dz  }|||z  z  }t                               d|› d�¦  «         t                               d|||z  z  › d�¦  «         |                     ||g¦  «         t                               d¦  «         t/          ¦   «          t9          | |||||
¦  «        \  }}|d                               ¦   «         }|j!        d         }|j"        }tG          |d¦  «        r|j$        n|j%        |j&        z  }tO          j(        || j)        tN          j*        ¬¦  «        }g }g }tW          j,        ¦   «         }||	k    �r©t;          | |d||¦  «        \  }}| -                    |¦  «         tW          j,        ¦   «         }|d         j!        d         dk    r–|d           .                    d¦  «        dz
  }| /                    d¬!¦  «         0                    d|j1        ¦  «         2                    |d|j1        ¦  «        }tO          j3        |d         d|¦  «         4                    ¦   «         }n|d         d d …dd d …f         }tO          j5        |d¬!¦  «        }||z  |j6        k    }| 7                    ||j6        ¦  «         8                    |dg¦  «        }tW          j,        ¦   «         } | -                    | |z
  ¦  «         tO          j9        ||gd¬!¦  «        }|dz  }||d<   tO          j9        |d          |  :                    tN          j;        ¦  «         8                    |d¦  «        gd¦  «        |d <   d"|v r<tO          j<        |d"         d¬!¦  «        d#          8                    |d¦  «        dz   |d"<   |d         j!        d         dk    r,|d         d d …d d…d d …f          =                    ¦   «         |d<   |d          >                    ¦   «          | j?        d$k    r|d%         |d%<   nû| j@        sôtƒ          |jB        ¦  «        D ](}!|d&|!› d'�         |d(|!› d'�<   |d&|!› d)�         |d(|!› d)�<   Œ)|d          j!        d         }"tƒ          |jB        ¦  «        D ]�}!tO          j(        |||"|| j)        | jC        ¬¦  «        }#tO          j(        |||"|| j)        | jC        ¬¦  «        }$| D                    d&|!› d'�|# =                    ¦   «         d&|!› d)�|$ =                    ¦   «         i¦  «         Œ�||	k    �°©tW          j,        ¦   «         }%| E                    d#¦  «         | jF        rzd*Š9t�          |¦  «        Š:t‘          |¦  «        }&t“          t•          ˆ9ˆ:fd+„|¦  «        ¦  «        }t‘          |¦  «        }'t                               d,|&|'z
  › d-‰9› d.‰:dz  › d/�¦  «         t]          |¦  «        t‘          |¦  «        z  }(|(dz  })|d|(z  z  }*t                               d0|)› d�¦  «         t                               d1|*› d�¦  «         |d#         }+|+dz  },|d|+z  z  }-t                               d2|,› d�¦  «         t                               d3|-› d�¦  «         | j        d4z  }.t]          |d |.…         ¦  «        t‘          |d |.…         ¦  «        z  }/|/dz  }0|d|/z  z  }1t                               d5|.› d6|0› d�¦  «         t                               d7|.› d6|1› d�¦  «         t]          |¦  «        t‘          |¦  «        z  }2|2dz  }3|d|2z  z  }4t                               d5| j        › d6|3› d�¦  «         t                               d7| j        › d6|4› d�¦  «         |%|z
  }5||| j        z   |5z  z  }6t                               d8|5› d9�¦  «         t                               d:||| j        z   |5z  z  › d�¦  «         t                               d;¦  «         |                     |)|*|,|-|0|1|3|4|5|6g
¦  «         | -                    |¦  «         �	Œ# t–          $ r.}7t                               d<|› d|› d=|7› �¦  «         Y d }7~7�	Œ6d }7~7ww xY wd>| j?        › d?t˜          jL         M                    ¦   «         d@›dA�}8t�          ||8| j        ¦  «         d S )BNFc                ó>   — d„ |                       ¦   «         D ¦   «         S )Nc                ó4   — i | ]\  }}t          |¦  «        |“ŒS rF   )r—   )Ú.0ÚkÚvs      rB   ú
<dictcomp>z*main.<locals>.<lambda>.<locals>.<dictcomp>j  s$   € Ð<]Ð<]Ð<]É4È1Èa½SÀ¹V¼VÀQÐ<]Ð<]Ð<]rd   )Úitems)Úds    rB   ú<lambda>zmain.<locals>.<lambda>j  s"   € Ð<]Ð<]ÐST×SZÒSZÑS\ÔS\Ð<]Ñ<]Ô<]€ rd   )Úobject_hookr   )r   r   r   zRunning batch size = z, prompt length = z2
                                A prompt of size z was not found in 'zv'. There are a couple of solutions to fix this.
                                1) You can change one of the keys in 'z' to be z).
                                    If za < actual prompt's length, the benchmark E2E tool will repeat the first word in the prompt until zB = actual prompt's length.
                                    If zm > actual prompt's length, the benchmark E2E tool will automatically trim the actual prompt's length so that zd = actual prompt's length.
                                2) You can add a new key-value entry in 'z' of the form 'z,': 'your prompt goes here'.
                zMeasuring prompt processing...iè  z&Average Latency of Prompt Processing: z msz)Average Throughput of Prompt Processing: z tpszMeasuring token generation...Ú	input_idséÿÿÿÿÚhead_dim)r,   Údtypeé   ÚlogitsÚattention_mask)ÚdimÚposition_idsr   r�   Úpast_key_valueszpresent.z.keyzpast_key_values.z.valueé
   c                ó   •— | ‰‰z  k     S r[   rF   )Úacc_timeÚanomaly_threshold_factorÚ
min_time_ss    €€rB   r°   zmain.<locals>.<lambda>  s   ø€ ¨HÐ7OÐR\Ñ7\Ò,\€ rd   zFiltered out z$ anomaly accelerator times that are zx greater than z ms...zAverage Latency of Sampling: z Average Throughput of Sampling: z"Latency of First Token Generated: z%Throughput of First Token Generated: rk   rj   z Tokens Generated: rl   zWall-Clock Latency: z szWall-Clock Throughput: zAdding results to CSVz$Could not benchmark at batch size = z - Ú
benchmark_Ú_e2e_z%Y-%m-%d_%H:%M:%Sz.csv)Or¦   r   rs   rt   Ú__dict__ÚopenÚprompts_fileÚjsonÚloadr   r/   r0   r1   r   r2   r3   r	   rC   Ú	itertoolsÚproductr    r¢   r—   r\   Úgeneration_lengthÚNotImplementedErrorÚtextwrapÚdedentrc   rY   Únum_runsÚextendÚcloneÚshapeÚnum_key_value_headsÚhasattrr´   Úhidden_sizeÚnum_attention_headsr-   Úzerosr5   ÚboolrL   rM   ÚappendÚsumÚ	unsqueezeÚrepeatÚ
vocab_sizeÚviewÚgatherÚsqueezeÚargmaxÚeos_token_idÚmasked_fillÚreshapeÚcatr4   Úint64ÚmaxÚ
contiguousÚzero_r]   rJ   rN   Únum_hidden_layersr   ÚupdateÚpopÚanomaly_filteringÚminÚlenÚlistÚfilterr6   ÚdatetimeÚnowry   );r   Úsize_to_promptÚfr_   r`   r>   Úall_csv_metricsÚ
batch_sizera   Ú
max_lengthrb   Úcsv_metricsrR   rS   Úaccelerator_prompt_latency_sÚaccelerator_prompt_latency_msÚaccelerator_prompt_thrptÚall_token_idsÚcurrent_lengthÚ	num_headsÚ	head_sizeÚhas_eosÚaccelerator_timesÚsampling_timesÚwall_clock_start_timeÚaccelerator_time_latency_sÚsampling_start_timeÚprompt_end_indicesÚidxsÚnext_token_logitsÚnext_tokensÚtokens_to_addÚsampling_end_timeÚiÚnew_sequence_lengthÚpresent_keyÚpresent_valueÚwall_clock_end_timeÚ	orig_sizeÚnew_sizeÚavg_sampling_latency_sÚavg_sampling_latency_msÚavg_sampling_thrptÚfirst_token_latency_sÚfirst_token_latency_msÚfirst_token_thrptÚhalfwayÚhalfway_token_latency_sÚhalfway_token_latency_msÚhalfway_token_thrptÚall_token_latency_sÚall_token_latency_msÚall_token_thrptÚwall_clock_latency_sÚwall_clock_thrptr@   rv   r¿   rÀ   s;                                                            @@rB   Úmainr"  b  sV  øø€ Ý‰:Œ:€DÝ�ÑÔÐÝ
‡K‚K�”ÑÔÐð €NÝ	ˆdÔÑ	 Ô	 ð _ AÝœ 1Ð2]Ð2]Ð^Ñ^Ô^ˆð_ð _ð _ñ _ô _ð _ð _ð _ð _ð _ð _øøøð _ð _ð _ð _õ Ô'Ø Ô,°Ò2Ð2ˆÔÐ¸¼Ø”.Ø”yØœ*ð	ñ ô €Fõ Ô-Ø Ô,°Ò2Ð2ˆÔÐ¸¼Ø”.Ø”yØœ*ð	ñ ô €Iõ �d‰OŒO€Eà€OÝ%.Ô%6°tÔ7GÈÔI\Ñ%]Ô%]ð ]tñ ]tÑ!ˆ
�MÝ$'¨
¡O¤OµS¸Ñ5GÔ5G�Mˆ
Ý�ŠÐY¨JÐYÐYÈ-ÐYÐYÑZÔZÐZÝ‰ŒˆØ" TÔ%;Ñ;ˆ
à Ð.Ð.Ý%Ý”ðØ2?ðð ØTXÔTeðð àGKÔGXðð àboðð ð )6ðð ð Yfðð ð )6ð	ð ð erð	ð ð
 KOÔJ[ðð ð
 mzðð ð ñô ñ
ô 
ð 
ð ! Ô/Ð0°:Ñ=ˆØ! =Ð1ˆðH	tå�KŠKÐ8Ñ9Ô9Ð9Ý9¸$ÀÀvÈyÐZgÐioÑpÔp‰OˆF�GÝ4AÀ$ÈÈtÌ}Ð^dÐfmÑ4nÔ4nÑ1Ð(¨'ð -IÈ4Ñ,OÐ)Ø'1°]ÐEaÑ5aÑ'bÐ$Ý�KŠKÐcÐA^ÐcÐcÐcÑdÔdÐdÝ�KŠKØ}¸JÈ-ÐZvÑJvÑ<wÐ}Ð}Ð}ñô ð ð ×ÒÐ =Ð?WÐXÑYÔYÐYõ �KŠKÐ7Ñ8Ô8Ð8Ý‰MŒMˆMÝ9¸$ÀÀvÈyÐZgÐioÑpÔp‰OˆF�Gà" ;Ô/×5Ò5Ñ7Ô7ˆMØ*Ô0°Ô4ˆNØÔ2ˆIå#*¨6°:Ñ#>Ô#>Ðt�”�ÀFÔDVÐZ`ÔZtÑDtð õ ”k *°TÔ5GÍuÌzÐZÑZÔZˆGð !#ÐØˆNå$(Ô$5Ñ$7Ô$7Ð!Ø  JÒ.Ñ.å6CÀDÈ%ÐQRÐTZÐ\cÑ6dÔ6dÑ3Ð*¨GØ!×(Ò(Ð)CÑDÔDÐDõ '+Ô&7Ñ&9Ô&9Ð#Ø˜8Ô$Ô*¨1Ô-°Ò1Ð1Ø)/Ð0@Ô)A×)EÒ)EÀaÑ)HÔ)HÈ1Ñ)LÐ&à*×4Ò4¸Ð4Ñ;Ô;ßš  6Ô#4Ñ5Ô5ßš˜j¨!¨VÔ->Ñ?Ô?ð õ
 ).¬°W¸XÔ5FÈÈ4Ñ(PÔ(P×(XÒ(XÑ(ZÔ(ZÐ%Ð%à(/°Ô(9¸!¸!¸!¸RÀÀÀ¸(Ô(CÐ%Ý#œlÐ+<À"ÐEÑEÔE�ð " KÑ/°9Ô3IÒI�ð !,× 7Ò 7¸ÀÔAWÑ XÔ X× `Ò `ÐblÐnoÐapÑ qÔ q�Ý$(Ô$5Ñ$7Ô$7Ð!Ø×%Ò%Ð&7Ð:MÑ&MÑNÔNÐNå %¤	¨=¸-Ð*HÈbÐ QÑ QÔ Q�Ø !Ñ#�ð '4��{Ñ#Ý+0¬9ØÐ,Ô-°°¯}ª}½U¼[Ñ/IÔ/I×/QÒ/QÐR\Ð^_Ñ/`Ô/`ÐaÐcdñ,ô ,�Ð'Ñ(ð " VÐ+Ð+Ý-2¬Y°v¸nÔ7MÐSTÐ-UÑ-UÔ-UÐVWÔ-X×-`Ò-`ÐakÐmnÑ-oÔ-oÐrsÑ-s�F˜>Ñ*ð ˜8Ô$Ô*¨1Ô-°Ò2Ð2Ø(/°Ô(9¸!¸!¸!¸R¸a¸RÀÀÀ¸(Ô(C×(NÒ(NÑ(PÔ(P�G˜HÑ%Ø˜Ô!×'Ò'Ñ)Ô)Ð)ð ”; $Ò&Ð&à07Ð8IÔ0J�FÐ,Ñ-Ð-ØÔ.ð å" 6Ô#;Ñ<Ô<ð ]ð ]˜Ø=DÐEWÐPQÐEWÐEWÐEWÔ=X˜Ð9°!Ð9Ð9Ð9Ñ:Ø?FÐG[ÐRSÐG[ÐG[ÐG[Ô?\˜Ð;°!Ð;Ð;Ð;Ñ<Ð<à*0Ð1AÔ*BÔ*HÈÔ*KÐ'Ý" 6Ô#;Ñ<Ô<ð ð ˜Ý&+¤kØ&Ø%Ø/Ø%Ø#'Ô#5Ø"&Ô"2ð'ñ 'ô '˜õ ).¬Ø&Ø%Ø/Ø%Ø#'Ô#5Ø"&Ô"2ð)ñ )ô )˜ð  Ÿšà 2¨1Ð 2Ð 2Ð 2°K×4JÒ4JÑ4LÔ4LØ 4¨1Ð 4Ð 4Ð 4°m×6NÒ6NÑ6PÔ6Pðñô ð ð ðQ ! JÒ.Ñ.õ^ #'Ô"3Ñ"5Ô"5Ðð ×!Ò! !Ñ$Ô$Ð$ØÔ%ð 
Ø+-Ð(Ý Ð!2Ñ3Ô3�
ÝÐ 1Ñ2Ô2�	Ý$(ÝÐ\Ð\Ð\Ð\Ð\Ð^oÑpÔpñ%ô %Ð!õ Ð0Ñ1Ô1�Ý—’ð a I°Ñ$8ð  að  aÐ^vð  að  að  HRð  UYñ  HYð  að  að  añô ð õ &)¨Ñ%8Ô%8½3¸~Ñ;NÔ;NÑ%NÐ"Ø&<¸tÑ&CÐ#Ø!+¨qÐ3IÑ/IÑ!JÐÝ�KŠKÐTÐ8OÐTÐTÐTÑUÔUÐUÝ�KŠKÐSÐ;MÐSÐSÐSÑTÔTÐTð %6°aÔ$8Ð!Ø%:¸TÑ%AÐ"Ø *¨aÐ2GÑ.GÑ HÐÝ�KŠKÐXÐ=SÐXÐXÐXÑYÔYÐYÝ�KŠKÐWÐ@QÐWÐWÐWÑXÔXÐXð Ô,°Ñ1ˆGÝ&)Ð*;¸H¸W¸HÔ*EÑ&FÔ&FÍÐM^Ð_gÐ`gÐ_gÔMhÑIiÔIiÑ&iÐ#Ø'>ÀÑ'EÐ$Ø",°Ð4KÑ0KÑ"LÐÝ�KŠKÐm°GÐmÐmÐPhÐmÐmÐmÑnÔnÐnÝ�KŠKÐl°wÐlÐlÐSfÐlÐlÐlÑmÔmÐmõ #&Ð&7Ñ"8Ô"8½3Ð?PÑ;QÔ;QÑ"QÐØ#6¸Ñ#=Ð Ø(¨AÐ0CÑ,CÑDˆOÝ�KŠKØp¨DÔ,BÐpÐpÐWkÐpÐpÐpñô ð õ �KŠKÐw°tÔ7MÐwÐwÐbqÐwÐwÐwÑxÔxÐxð $7Ð9NÑ#NÐ Ø)¨m¸dÔ>TÑ.TÐXlÑ-lÑmÐÝ�KŠKÐGÐ/CÐGÐGÐGÑHÔHÐHÝ�KŠKØ~¨*¸ÈÔI_Ñ9_ÐcwÑ8wÑ*xÐ~Ð~Ð~ñô ð õ
 �KŠKÐ/Ñ0Ô0Ð0Ø×Òà+Ø&Ø*Ø%Ø,Ø'Ø(Ø#Ø(Ø$ðñô ð ð ×"Ò" ;Ñ/Ô/Ð/Ñ/øåð 	tð 	tð 	tÝ�KŠKÐr¸zÐrÐrÐ]jÐrÐrÐopÐrÐrÑsÔsÐsÐsÐsÐsÐsÑsøøøøð	tøøøð ^˜DœKÐ]Ð]­hÔ.?×.CÒ.CÑ.EÔ.EÐ]Ð]Ð]Ð]€HÝ� (¨DÔ,BÑCÔCÐCÐCÐCs+   ÁA9Á9A=Â A=Ç0ah3è3
i+è=#i&é&i+Ú__main__)r   r   )&Ú
__future__r   r�   rñ   rf   rÈ   rÆ   Úloggingr”   rÌ   rL   Únumpyrš   Úpandasrp   r-   Úbenchmark_helperr   Úllama_inputsr   r   Útransformersr   r   r	   r
   Úonnxruntimer:   Ú	getLoggerÚ__name__rs   rC   rY   rc   r\   ry   r¦   r"  rF   rd   rB   ú<module>r.     s™  ðð@ #Ð "Ð "Ð "Ð "Ð "à €€€Ø €€€Ø 	€	€	€	Ø Ð Ð Ð Ø €€€Ø €€€Ø 	€	€	€	Ø €€€Ø €€€à Ð Ð Ð Ø Ð Ð Ð Ø €€€Ø )Ð )Ð )Ð )Ð )Ð )Ø SÐ SÐ SÐ SÐ SÐ SÐ SÐ SØ \Ð \Ð \Ð \Ð \Ð \Ð \Ð \Ð \Ð \Ð \Ð \à Ð Ð Ð à	ˆÔ	˜8Ñ	$Ô	$€ð=ð =ð =ð =ð@ð ð ð>ð ð ðð ð ð
1ð 1ð 1ð2að að aðHzDð zDð zDðz ˆzÒÐØ€D�F„F€F€F€Fð Ðrd   