§
    û¸¸jø.  ã                   óæ  — d dl Z d dlZd dlZd dlZ	 d dlmZmZ n# e$ r d dlmZmZ Y nw xY wd dl	m
Z
 d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZmZ  ej        e¦  «         ej        d¦  «        k     r ede› d�¦  «        ‚g d¢Zdefd„Z	 ddededz  defd„Z de!e"eef                  defd„Z#de!e"eef                  dedefd„Z$d„ Z%d d„Z&e'dk    r e%¦   «          dS dS )!é    N)ÚPackageNotFoundErrorÚversion)ÚPath)ÚAny)Úload_dataset)Ú	evaluator)ÚORTModelForQuestionAnswering)Ú__version__)r   )ÚAutoTokenizerÚpipelinez1.13.1z1Please install optimum>=1.13.1. Current version: ú.)z5bert-large-uncased-whole-word-masking-finetuned-squadzdeepset/roberta-base-squad2z%distilbert-base-cased-distilled-squadÚpackage_namec                 óD   — 	 t          | ¦  «        S # t          $ r Y d S w xY w©N)r   r   )r   s    ú/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/onnxruntime/transformers/models/bert/eval_squad.pyÚget_package_versionr   7   s7   € ðÝ�|Ñ$Ô$Ð$øÝð ð ð Øˆtˆtðøøøs   ‚ ‘
žÚCUDAExecutionProviderFÚmodel_idÚ	onnx_pathÚuse_io_bindingc                 ó˜  — |€~t          j        | d||¬¦  «        }t          j                             d| ¦  «        }|                     |¦  «         t          j                             |d¦  «        }t          d|¦  «         nGt          j        t          j                             |¦  «        t          |¦  «        j	        ||¬¦  «        }||fS )a¨  Load onnx model given pretrained model name and optional ONNX model path. If onnx_path is None,
    the default onnx model from optimum will be used.

    Args:
        model_id (str): pretrained model name or checkpoint path
        onnx_path (Optional[str], optional): path of onnx model to evaluate. Defaults to None.

    Returns:
        model: ORTModel for the onnx model
        onnx_path: the path of onnx model
    NT)ÚexportÚproviderr   r   z
model.onnxzModel is exported to onnx file:)Ú	file_namer   r   )
r	   Úfrom_pretrainedÚosÚpathÚjoinÚsave_pretrainedÚprintÚdirnamer   Úname)r   r   r   r   ÚmodelÚsave_onnx_dirs         r   Úload_onnx_modelr%   >   sÅ   € ð Ðå,Ô<Ø˜T¨HÀ^ð
ñ 
ô 
ˆõ œŸš S¨(Ñ3Ô3ˆØ×Ò˜mÑ,Ô,Ð,Ý”G—L’L °Ñ=Ô=ˆ	ÝÐ/°Ñ;Ô;Ð;Ð;å,Ô<ÝŒG�OŠO˜IÑ&Ô&Ý˜9‘o”oÔ*ØØ)ð	
ñ 
ô 
ˆð �)ÐÐó    ÚresultsÚcsv_filenamec                 ó8  — t          |ddd¬¦  «        5 }g d¢}t          j        ||¬¦  «        }|                     ¦   «          | D ]}|                     |¦  «         Œ|                     ¦   «          ddd¦  «         n# 1 swxY w Y   t          d|› �¦  «         dS )	zµOutput a CSV file with detail of each test results.

    Args:
        results (List[Dict[str, Any]]): list of JSON results.
        csv_filename (str): path of output CSV file
    ÚaÚ Úascii©ÚmodeÚnewlineÚencoding)Úpretrained_model_namer   r   Údisable_fused_attentionÚ
batch_sizeÚsequence_lengthr   ÚexactÚf1ÚtotalÚHasAns_exactÚ	HasAns_f1ÚHasAns_totalÚ
best_exactÚbest_exact_threshÚbest_f1Úbest_f1_threshÚtotal_time_in_secondsÚsamples_per_secondÚlatency_in_seconds©Ú
fieldnamesNz&Detail results are saved to csv file: )ÚopenÚcsvÚ
DictWriterÚwriteheaderÚwriterowÚflushr    )r'   r(   Úcsv_fileÚcolumn_namesÚ
csv_writerÚresults         r   Úoutput_detailsrN   b   sû   € õ 
ˆl ¨b¸7Ð	CÑ	CÔ	Cð Àxð
ð 
ð 
ˆõ. ”^ H¸ÐFÑFÔFˆ
Ø×ÒÑ Ô Ð Øð 	(ð 	(ˆFØ×Ò Ñ'Ô'Ð'Ð'à�ŠÑÔÐð;ð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð õ> 
Ð
A°<Ð
AÐ
AÑBÔBÐBÐBÐBs   ”AA=Á=BÂBÚmetric_namec           	      ó  ‡— t          |ddd¬¦  «        5 }g d¢Št          d„ | D ¦   «         ¦  «        }|                     ¦   «          t          d„ | D ¦   «         ¦  «        }|                     ¦   «          t          d„ | D ¦   «         ¦  «        }|                     ¦   «          g }|D ]"}|D ]}	|                     d	|	› d
|› �¦  «         ŒŒ#t	          j        |‰|z   ¬¦  «        }
|
                     ¦   «          |D ]Ñ}i }i }|                     d„ |D ¦   «         ¦  «         | D ]t}|d         |k    rf||         r^ˆfd„|                     ¦   «         D ¦   «         }|s|                     |¦  «         |d         }	|d         }d	|	› d
|› �}||v r||         ||<   Œu|r3|D ]}| 	                    |d¦  «        ||<   Œ|
 
                    |¦  «         ŒÒ|                     ¦   «          ddd¦  «         n# 1 swxY w Y   t          d|› d|› �¦  «         dS )a  Output a CSV file with summary of a metric on combinations of batch_size and sequence_length.

    Args:
        results (List[Dict[str, Any]]): list of JSON results.
        csv_filename (str): path of output CSV file
        metric_name (str): the metric to summarize
    r*   r+   r,   r-   )r1   r   r   r2   r   c                 ó   — h | ]
}|d          ’ŒS )r   © ©Ú.0rM   s     r   ú	<setcomp>z!output_summary.<locals>.<setcomp>œ   s   € ÐEÐEÐE°6˜6 +Ô.ÐEÐEÐEr&   c                 ó   — h | ]
}|d          ’ŒS )r3   rR   rS   s     r   rU   z!output_summary.<locals>.<setcomp>Ÿ   s   € ÐGÐGÐG°V˜F <Ô0ÐGÐGÐGr&   c                 ó   — h | ]
}|d          ’ŒS )r4   rR   rS   s     r   rU   z!output_summary.<locals>.<setcomp>¢   s   € Ð QÐ QÐ Q¸v Ð(9Ô!:Ð QÐ QÐ Qr&   ÚbÚ_srB   c                 ó   — i | ]}|d “ŒS )r+   rR   )rT   Úks     r   ú
<dictcomp>z"output_summary.<locals>.<dictcomp>³   s   € Ð4Ð4Ð4 Q˜1˜bÐ4Ð4Ð4r&   r   c                 ó$   •— i | ]\  }}|‰v ¯	||“ŒS rR   rR   )rT   r[   ÚvÚheader_namess      €r   r\   z"output_summary.<locals>.<dictcomp>·   s)   ø€ ÐTÐTÐT©¨¨1À!À|ÐBSÐBS˜q !ÐBSÐBSÐBSr&   r3   r4   NzSummary results for z are saved to csv file: )rD   ÚlistÚsortÚappendrE   rF   rG   ÚupdateÚitemsÚgetrH   rI   r    )r'   r(   rO   rJ   Ú
model_listÚbatch_sizesÚsequence_lengthsÚ	key_namesr4   r3   rL   r#   ÚrowÚvaluesrM   ÚheadersÚkeyr_   s                    @r   Úoutput_summaryrn   ‹   sî  ø€ õ 
ˆl ¨b¸7Ð	CÑ	CÔ	Cð 4Àxð
ð 
ð 
ˆõ ÐEÐE¸WÐEÑEÔEÑFÔFˆ
Ø�ŠÑÔÐåÐGÐG¸wÐGÑGÔGÑHÔHˆØ×ÒÑÔÐåÐ QÐ QÈÐ QÑ QÔ QÑRÔRÐØ×ÒÑÔÐàˆ	Ø/ð 	Fð 	FˆOØ)ð Fð F�
Ø× Ò Ð!D ZÐ!DÐ!D°?Ð!DÐ!DÑEÔEÐEÐEðFõ ”^ H¸È	Ñ9QÐRÑRÔRˆ
Ø×ÒÑ Ô Ð àð 	)ð 	)ˆEØˆCð ˆFØ�MŠMÐ4Ð4¨)Ð4Ñ4Ô4Ñ5Ô5Ð5à!ð :ð :�Ø˜+Ô&¨%Ò/Ð/°F¸;Ô4GÐ/ØTÐTÐTÐT°·²±´ÐTÑTÔT�GØð ,ØŸ
š
 7Ñ+Ô+Ð+à!'¨Ô!5�JØ&,Ð->Ô&?�OØ=˜jÐ=Ð=¨OÐ=Ð=�Cà˜iÐ'Ð'Ø&,¨[Ô&9˜˜s™øàð )Ø$ð 3ð 3�CØ%Ÿzšz¨#¨rÑ2Ô2�C˜‘H�HØ×#Ò# CÑ(Ô(Ð(øà�ŠÑÔÐði4ð 4ð 4ñ 4ô 4ð 4ð 4ð 4ð 4ð 4ð 4øøøð 4ð 4ð 4ð 4õl 
Ð
T Ð
TÐ
TÀlÐ
TÐ
TÑUÔUÐUÐUÐUs   •GG)Ç)G-Ç0G-c                  ó@  — t          ¦   «         } t          | ¦  «         dD ]&}t          |¦  «        }|rt          |› d�|¦  «         Œ'| j        }| j        r;t
          j                             | j        ¦  «        st          d| j        › �¦  «        ‚t
          j	         
                    dd¦  «        dk    }g }t          j        |¦  «        }| j        D �]"}||_        t          |dz  d¦  «        |_        | j        €t          d	¦  «         t#          j        ¦   «         }t%          || j        | j        | j        ¦  «        \  }	}
t#          j        ¦   «         |z
  }t          d
|d›d�¦  «         t          |	j        ¦  «         ||	j        j        k    rt          d¦  «        ‚t/          d|	|d| j        ¬¦  «        }t3          d¦  «        }t          d¦  «         t#          j        ¦   «         }t5          d| j        dk    rd| j        › d�nd¬¦  «        }t#          j        ¦   «         |z
  }t          d|d›d�¦  «         t          d¦  «         t#          j        ¦   «         }|                     ||dd¬¦  «        }t#          j        ¦   «         |z
  }t          d|d›d�¦  «         | j        |d<   ||d<   ||d<   |
|d <   | j        |d!<   ||d"<   | j        |d#<   t          |¦  «         |                     |¦  «         �Œ$t=          |d$¦  «         d%D ]}t?          ||› d&�|¦  «         Œd S )'N)zonnxruntime-gpuÚonnxruntimeÚonnxÚtorchÚtransformersÚoptimumÚdatasetsÚevaluatez versionz Onnx model path does not exist: ÚORT_DISABLE_FUSED_ATTENTIONÚ0Ú1é   é€   z4Exporting onnx model. It might take a few minutes...z!Onnx model exported or loaded in z.1fz secondszTsequence length should not be larger than {ort_model.config.max_position_embeddings}zquestion-answeringT)r#   Ú	tokenizerÚquestion_firstr3   zLoading dataset...Úsquadr   zvalidation[:ú]Ú
validation)ÚsplitzDataset loaded in z<Evaluating squad_v2 with ORT. It might take a few minutes...Úsquad_v2)Úmodel_or_pipelineÚdataÚmetricÚsquad_v2_formatzEvaluation done in r   r2   r1   r   r3   r4   r   z
detail.csv)r6   r5   r@   z.csv) Úparse_argumentsr    r   Ú
model_namerq   r   r   ÚexistsÚRuntimeErrorÚenvironre   r   r   rh   Úmodel_max_lengthÚminÚ
doc_strideÚtimer%   r   r   ÚconfigÚmax_position_embeddingsr   r3   r   r   r7   Úcomputerb   rN   rn   )Úargsr"   Úpackage_versionr1   r2   Úall_resultsr|   r4   Ú
start_timeÚ	ort_modelr   ÚlatencyÚqa_pipelineÚtask_evaluatorÚsquad_datasetrM   rO   s                    r   Úmainrœ   Ì   s~  € ÝÑÔ€DÝ	ˆ$�K„K€Kàvð 6ð 6ˆÝ-¨dÑ3Ô3ˆØð 	6Ý�TÐ#Ð#Ð# _Ñ5Ô5Ð5øà œOÐØ„yð K�œŸš¨¬	Ñ2Ô2ð KÝÐI¸d¼iÐIÐIÑJÔJÐJå œjŸnšnÐ-JÈCÑPÔPÐTWÒWÐà€KÝÔ-Ð.CÑDÔD€IØÔ0ð -#ñ -#ˆØ%4ˆ	Ô"Ý" ?°aÑ#7¸Ñ=Ô=ˆ	ÔØŒ9ÐÝÐHÑIÔIÐIÝ”Y‘[”[ˆ
Ý.Ð/DÀdÄiÐQUÔQ^Ð`dÔ`sÑtÔtÑˆ	�9Ý”)‘+”+ 
Ñ*ˆÝÐG°'ÐGÐGÐGÐGÑHÔHÐHåˆiÔÑÔÐØ˜YÔ-ÔEÒEÐEÝÐuÑvÔvÐvåØ ¨	¸YÐW[ÐhlÔhwð
ñ 
ô 
ˆõ #Ð#7Ñ8Ô8ˆÝÐ"Ñ#Ô#Ð#Ý”Y‘[”[ˆ
Ý$ WÐTXÔT^ÐabÒTbÐTbÐ4PÀ4Ä:Ð4PÐ4PÐ4PÐ4PÐhtÐuÑuÔuˆÝ”)‘+”+ 
Ñ*ˆÝÐ8 7Ð8Ð8Ð8Ð8Ñ9Ô9Ð9åÐLÑMÔMÐMÝ”Y‘[”[ˆ
Ø×'Ò'Ø)ØØØ ð	 (ñ 
ô 
ˆõ ”)‘+”+ 
Ñ*ˆÝÐ9 GÐ9Ð9Ð9Ð9Ñ:Ô:Ð:à!œ]ˆˆzÑØ,CˆÐ(Ñ)Ø*?ˆÐ&Ñ'Ø'ˆˆ{ÑØ#œˆˆ|ÑØ$3ˆÐ Ñ!Ø#'Ô#6ˆÐÑ Ýˆf‰Œˆà×Ò˜6Ñ"Ô"Ð"Ñ"å�; Ñ-Ô-Ð-à<ð Gð GˆÝ�{ {Ð$8Ð$8Ð$8¸+ÑFÔFÐFÐFðGð Gr&   c           	      óH  — t          j        ¦   «         }|                     dddt          t          d         dt          › �¬¦  «         |                     ddd	t
          d
gd¬¦  «         |                     ddt
          dd¬¦  «         |                     ddt
          dd¬¦  «         |                     ddt          d d¬¦  «         |                     dddd¬¦  «         |                     dddd¬¦  «         |                     d¬¦  «         |                     | ¦  «        }|S ) Nz-mz--model_nameFr   z=Checkpoint directory or pre-trained model names in the list: )ÚrequiredÚtypeÚdefaultÚhelpz-sz--sequence_lengthsú+i€  zFSequence lengths for onnx model inputs. It could have multiple values.)ÚnargsrŸ   r    r¡   z-bz--batch_sizeé   zbatch size for inference.)rŸ   r    r¡   z-tz--totalz+Total samples to test. 0 means all samples.z--onnxzbOptional onnx model path. If not specified, optimum will be used to export onnx model for testing.z
--providerr   zRSelect which Execution Provider to use for runs. Default is CUDAExecutionProvider.)rž   r    r¡   z--use_io_bindingÚ
store_truezUse IO Binding for GPU.)rž   Úactionr¡   )r   )ÚargparseÚArgumentParserÚadd_argumentÚstrÚPRETRAINED_SQUAD_MODELSÚintÚset_defaultsÚ
parse_args)ÚargvÚparserr“   s      r   r‡   r‡     sw  € ÝÔ$Ñ&Ô&€Fà
×ÒØØØÝÝ'¨Ô*ØfÕMdÐfÐfð ñ ô ð ð ×ÒØØØÝØ�ØUð ñ ô ð ð ×ÒØØÝØØ(ð ñ ô ð ð ×Ò˜˜i­c¸1ÐCpÐÑqÔqÐqà
×ÒØØÝØØqð ñ ô ð ð ×ÒØØØ'Øað	 ñ ô ð ð ×ÒÐ*°UÀ<ÐVoÐÑpÔpÐpØ
×Ò uÐÑ-Ô-Ð-à×Ò˜TÑ"Ô"€Dà€Kr&   Ú__main__)Nr   Fr   )(r§   rE   r   r�   Úimportlib.metadatar   r   ÚImportErrorÚimportlib_metadataÚpathlibr   Útypingr   ru   r   rv   r   Úoptimum.onnxruntimer	   Úoptimum.versionr
   Úoptimum_versionÚ	packagingÚversion_checkrs   r   r   Úparser«   rª   r   Úboolr%   r`   ÚdictrN   rn   rœ   r‡   Ú__name__rR   r&   r   ú<module>rÀ      s‘  ðð2 €€€Ø 
€
€
€
Ø 	€	€	€	Ø €€€ðAØ@Ð@Ð@Ð@Ð@Ð@Ð@Ð@Ð@øØð Að Að AØ@Ð@Ð@Ð@Ð@Ð@Ð@Ð@Ð@Ð@ðAøøøð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à !Ð !Ð !Ð !Ð !Ð !Ø Ð Ð Ð Ð Ð Ø <Ð <Ð <Ð <Ð <Ð <Ø :Ð :Ð :Ð :Ð :Ð :Ø .Ð .Ð .Ð .Ð .Ð .Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0à€=Ô�Ñ'Ô'Ð*=¨-Ô*=¸hÑ*GÔ*GÒGÐGØ
ˆ+Ð\È/Ð\Ð\Ð\Ñ
]Ô
]Ð]ðð ð Ð ð cð ð ð ð ð kpð!ð !Øð!Ø! D™jð!Øcgð!ð !ð !ð !ðH&C˜D  c¨3 h¤Ô0ð &CÀð &Cð &Cð &Cð &CðR>V˜D  c¨3 h¤Ô0ð >VÀð >VÐRUð >Vð >Vð >Vð >VðBCGð CGð CGðL3ð 3ð 3ð 3ðl ˆzÒÐØ€D�F„F€F€F€Fð Ðs   ’ ›+ª+