§
    û¸¸jÔ,  ã                  óž  — d dl mZ d dlZd dlZd dlZd dlZd dlZd dlm	Z
 d dlZd dlZd dlmZ d dlmZmZ d dlmZmZmZmZmZmZ d dlmZ d dlmZ d dlmZ d d	lmZ d dl Z! ej"        d
¦  «        Z#d"d„Z$d"d„Z%d„ Z&	 	 d#d$d„Z'd%d„Z(g fd%d„Z)e*d k    r8d!Z+ej,         +                    e+¦  «          ej-        e+¦  «          e)¦   «          dS dS )&é    )ÚannotationsN)Úsetup_logger)Úget_rankÚget_size)Úadd_io_bindings_as_ortvaluesÚconvert_inputs_for_ortÚ%get_merged_sample_with_past_kv_inputsÚget_sample_inputsÚget_sample_with_past_kv_inputsÚverify_ort_inputs)Úsetup_torch_model)Úmake_dynamic_cache)Ú
AutoConfig)ÚDynamicCacheÚ Úargsúargparse.NamespaceÚconfigr   c                ó6   — | j         rdnd\  }}|j        }|||fS )N)é   é   )r   r   )Úuse_past_kvÚmax_position_embeddings)r   r   Úpast_sequence_lengthÚcurr_sequence_lengthÚmax_sequence_lengths        ú‚/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/onnxruntime/transformers/models/llama/llama_parity.pyÚget_sequence_lengthsr   %   s3   € Ø;?Ô;KÐ1W°°ÐQWÑ.ÐÐ.Ø Ô8ÐØÐ!5Ð7JÐJÐJó    c                ó0  — t          ¦   «         }d}t          | |¦  «        \  }}}| j        r)t          || j        ||||| j        | j        d|¬¦
  «
        }nA| j        r!t          || j        ||| j        d|¬¦  «        }nt          || j        ||d¬¦  «        }|S )Né   T)Úseq_lenÚpast_seq_lenÚmax_seq_lenÚuse_fp16Úuse_buffer_shareÚreturn_dictÚ
world_size)r%   r'   r(   )r'   )
r   r   Úmergedr	   Údevicer%   r&   r   r   r
   )r   r   r(   Ú
batch_sizer   Úsequence_lengthr   Úinputss           r   Ú
get_inputsr.   +   sÏ   € å‘”€JØ€JÝAUÐVZÐ\bÑAcÔAcÑ>Ð˜/Ð+>à„{ð gÝ6ØØŒKØØ#Ø-Ø+Ø”]Ø!Ô2ØØ!ð
ñ 
ô 
ˆˆð 
Ô	ð gÝ/ØØŒKØØØ”]ØØ!ð
ñ 
ô 
ˆˆõ # 6¨4¬;¸
ÀOÐaeÐfÑfÔfˆà€Mr   c                ó"  — t          | t          t          t          f¦  «        r| S t          | t          ¦  «        rt	          d„ | D ¦   «         ¦  «        S t          | t
          ¦  «        rd„ | D ¦   «         S t          | t          ¦  «        rd„ | D ¦   «         S t          | t          ¦  «        rd„ |                      ¦   «         D ¦   «         S t          | t          j
        ¦  «        r|                      ¦   «         S t          | d¦  «        r|                      ¦   «         S t          | t          ¦  «        rCt          t!          t          t#          | j        | j        d¬¦  «        ¦  «        ¦  «        ¦  «        S t)          dt+          | ¦  «        › �¦  «        ‚)	Nc              3  ó4   K  — | ]}t          |¦  «        V — Œd S )N©Útorch_deepcopy©Ú.0Úvs     r   ú	<genexpr>z!torch_deepcopy.<locals>.<genexpr>R   s*   è è € Ð6Ð6¨1•^ AÑ&Ô&Ð6Ð6Ð6Ð6Ð6Ð6r   c                ó,   — g | ]}t          |¦  «        ‘ŒS © r1   r3   s     r   ú
<listcomp>z"torch_deepcopy.<locals>.<listcomp>T   ó    € Ð1Ð1Ð1 a•˜qÑ!Ô!Ð1Ð1Ð1r   c                ó,   — h | ]}t          |¦  «        ’ŒS r8   r1   r3   s     r   ú	<setcomp>z!torch_deepcopy.<locals>.<setcomp>V   r:   r   c                ó4   — i | ]\  }}|t          |¦  «        “ŒS r8   r1   )r4   Úkr5   s      r   ú
<dictcomp>z"torch_deepcopy.<locals>.<dictcomp>X   s&   € Ð?Ð?Ð?©¨¨A�•> !Ñ$Ô$Ð?Ð?Ð?r   ÚcloneF)Ústrictz(torch_deepcopy not implemented for type )Ú
isinstanceÚintÚfloatÚstrÚtupleÚlistÚsetÚdictÚitemsÚnpÚndarrayÚcopyÚhasattrr@   r   r   r2   ÚzipÚ	key_cacheÚvalue_cacheÚNotImplementedErrorÚtype)Úvalues    r   r2   r2   N   sq  € Ý�%�#�u¥cÐ*Ñ+Ô+ð ØˆÝ�%�ÑÔð 7ÝÐ6Ð6°Ð6Ñ6Ô6Ñ6Ô6Ð6Ý�%�ÑÔð 2Ø1Ð1¨5Ð1Ñ1Ô1Ð1Ý�%�ÑÔð 2Ø1Ð1¨5Ð1Ñ1Ô1Ð1Ý�%�ÑÔð @Ø?Ð?°·²±´Ð?Ñ?Ô?Ð?Ý�%�œÑ$Ô$ð Ø�zŠz‰|Œ|ÐÝˆu�gÑÔð Ø�{Š{‰}Œ}ÐÝ�%�Ñ&Ô&ð oÝ!¥.µµc¸%¼/È5ÔK\ÐejÐ6kÑ6kÔ6kÑ1lÔ1lÑ"mÔ"mÑnÔnÐnõ ÐVÍÈeÉÌÐVÐVÑ
WÔ
WÐWr   ÚlocationrE   Úuse_auth_tokenÚboolÚkv_cache_ortvaluesrI   Úpytorch_modelúNone | torch.nn.ModuleúNone | AutoConfigc                óÔ  — |}|€9t          | ||| j        rt          j        nt          j        | j        ¬¦  «        \  }}t          | |¦  «        }d|v rLt          j        t          j
        ¦  «        t          j        d¦  «        k    rt          |d         ¦  «        |d<   | j        dk    rt          j                             ¦   «          t          j        ¦   «         } |di t!          |¦  «        ¤Žj                             ¦   «                              ¦   «                              ¦   «         }	| j        dk    rt          j                             ¦   «          t          j        ¦   «         }
t*                               d|
|z
  › d�¦  «         | j        r!|�~t          j                             ¦   «          t3          | |¦  «        \  }}}t5          || j        ||¬¦  «        }| j                             ¦   «         › d�}|d	k    r|d
| j        if}t=          j        | j         t=          j!        ¦   «         |g¬¦  «        }tE          ||¦  «        }| j        dk    r³tG          ||| j        tI          | j        ¦  «        | j        |¬¦  «        \  }}| %                    ¦   «          t          j        ¦   «         }| &                    |¦  «         | '                    ¦   «          t          j        ¦   «         }
| (                    ¦   «         d         }~nDt          j        ¦   «         }| )                    d |¦  «        }t          j        ¦   «         }
|d         }t*                               d|
|z
  › d�¦  «         d| j         v s	d| j         v rdnd}tU          j+        |	|||¬¦  «        }t*           ,                    d|› �¦  «         |s2t*           ,                    dtU          j-        |	|z
  ¦  «        › �¦  «         |S )N©Útorch_dtyper*   Úpast_key_valuesz4.45ÚcpuzPyTorch took z s)r&   r#   r$   ÚExecutionProviderÚCUDAExecutionProviderÚ	device_id)Úsess_optionsÚ	providers)Ú
ort_inputsr*   rc   r&   rX   r   zONNX Runtime took Úint4Úint8g      4@g      à?)ÚrtolÚatolz,Are PyTorch and ONNX Runtime results close? z
Max diff: r8   ).r   r%   ÚtorchÚfloat16Úfloat32r*   r.   ÚpvÚVersionÚtransformersÚ__version__r   Úexecution_providerÚcudaÚsynchronizeÚtimer2   ÚlogitsÚdetachr`   ÚnumpyÚloggerÚinfoÚ	small_gpuÚempty_cacher   r   r&   ÚupperÚrankÚortÚInferenceSessionÚonnx_model_pathÚSessionOptionsr   r   rC   Úsynchronize_inputsÚrun_with_iobindingÚsynchronize_outputsÚcopy_outputs_to_cpuÚrunrK   ÚallcloseÚwarningÚmax)r   rU   rV   rX   rY   r   Úpy_modelr-   Ú
start_timeÚ
pt_outputsÚend_timer   Ú_r   ÚepÚ	ort_modelÚ
io_bindingÚort_outputsÚtolÚparitys                       r   Úverify_parityr–   d   sá  € ð €HØÐÝ,ØØØØ*.¬-ÐJ�œ˜½U¼]Ø”;ð
ñ 
ô 
Ñˆ�õ ˜˜fÑ%Ô%€Fà˜FÐ"Ð"¥r¤zµ,Ô2JÑ'KÔ'KÍrÌzÐZ`ÑOaÔOaÒ'aÐ'aå$6°vÐ>OÔ7PÑ$QÔ$QˆÐ Ñ!ð Ô %Ò'Ð'ÝŒ
×ÒÑ Ô Ð Ý”‘”€Jð �Ð3Ð3�N¨6Ñ2Ô2Ð3Ð3Ô:×AÒAÑCÔC×GÒGÑIÔI×OÒOÑQÔQ€JØÔ %Ò'Ð'ÝŒ
×ÒÑ Ô Ð ÝŒy‰{Œ{€HÝ
‡K‚KÐ9 ¨:Ñ 5Ð9Ð9Ð9Ñ:Ô:Ð:à„~ð !˜(Ð.ØÝŒ
×ÒÑ Ô Ð õ 4HÈÈfÑ3UÔ3UÑ0Ð˜!Ð0Ý#ØØÔ.Ø)Ø'ð	ñ ô €Fð Ô#×)Ò)Ñ+Ô+Ð	>Ð	>Ð	>€BØ	Ð$Ò$Ð$Ø�; ¤	Ð*Ð+ˆÝÔ$ØÔÝÔ'Ñ)Ô)Ø�$ðñ ô €Iõ
 ˜y¨&Ñ1Ô1€Fð Ô %Ò'Ð'Ý)EØØØÔ*Ý˜$œ)‘n”nØ!Ô2Ø1ð*
ñ *
ô *
Ñ&ˆ
Ð&ð 	×%Ò%Ñ'Ô'Ð'Ý”Y‘[”[ˆ
Ø×$Ò$ ZÑ0Ô0Ð0Ø×&Ò&Ñ(Ô(Ð(Ý”9‘;”;ˆà ×4Ò4Ñ6Ô6°qÔ9ˆØˆIõ ”Y‘[”[ˆ
Ø—m’m D¨&Ñ1Ô1ˆÝ”9‘;”;ˆà! !”nˆå
‡K‚KÐ> X°
Ñ%:Ð>Ð>Ð>Ñ?Ô?Ð?ð ˜4Ô/Ð/Ð/°6¸TÔ=QÐ3QÐ3Qˆ#ˆ#ÐW[€CÝŒ[˜ [°sÀÐEÑEÔE€FÝ
‡N‚NÐJÀ&ÐJÐJÑKÔKÐKØð HÝ�ŠÐF¥B¤F¨:¸Ñ+CÑ$DÔ$DÐFÐFÑGÔGÐGØÐr   Úargvú	list[str]c                ó>  — t          j        ¦   «         }|                     dddd¬¦  «         |                     dddt          j                             d¦  «        d	¬
¦  «         |                     dddt          j                             d¦  «        d¬
¦  «         |                     ddddg d¢d¬¦  «         |                     dddd¬¦  «         |                     d¬¦  «         |                     dddd¬¦  «         |                     d¬¦  «         |                     dd dd!¬¦  «         |                     d¬"¦  «         |                     d#dd$¬¦  «         |                     d¬%¦  «         |                     d&d'dg d(¢d)¬*¦  «         |                     d+dt          d,d-¬.¦  «         |                     d/dd0¬¦  «         | g k    r|                     ¦   «         n|                     | ¦  «        }|j	        d1v s|j	        d2k    r|j
        dk    rd3nd4|_	        |S )5Nz-mz--model_nameFzModel name in Hugging Face)ÚrequiredÚhelpz-tz--torch_model_directoryú.zMPath to folder containing PyTorch model and associated files if saved on disk)rš   Údefaultr›   z-oz--onnx_model_pathTzSPath to ONNX model (with external data files saved in the same folder as the model)z-epz--execution_providerr`   )r`   rs   Úrocmz(Execution provider to verify parity with)rš   r�   Úchoicesr›   z-vz	--verboseÚ
store_truezPrint verbose logs)Úactionr›   )Úverbosez-pz--use_past_kvzfUse past key and past value as inputs to the model. Necessary for decoder_with_past_model.onnx models.)r   z-gz--use_buffer_sharezWUse if model has GroupQueryAttention and you want to enable past-present buffer sharing)r&   z--mergedz2Use merged model (i.e. decoder_merged_model.onnx).)r)   z-fpz--precision)rg   rh   Úfp16Úfp32zPrecision of model)rš   rŸ   r›   z--cache_dirz./model_cachezQmodel cache dir to override default HF cache dir to avoid overflood the /home dir)rš   rS   r�   r›   z--small_gpuzhLoad the llama in GPU every time for parity_check if it's running in a machine which GPU memory < 36GB. >   r¤   rh   rg   r¤   r£   )ÚargparseÚArgumentParserÚadd_argumentÚosÚpathÚjoinÚset_defaultsrE   Ú
parse_argsÚ	precisionrr   )r—   Úparserr   s      r   Úget_argsr¯   Ç   sç  € ÝÔ$Ñ&Ô&€Fà
×ÒØØØØ)ð	 ñ ô ð ð ×ÒØØ!ØÝ”—’˜SÑ!Ô!Ø\ð ñ ô ð ð ×ÒØØØÝ”—’˜SÑ!Ô!Øbð ñ ô ð ð ×ÒØØØØØ'Ð'Ð'Ø7ð ñ ô ð ð ×ÒØØØØ!ð	 ñ ô ð ð ×Ò ÐÑ&Ô&Ð&à
×ÒØØØØuð	 ñ ô ð ð ×Ò EÐÑ*Ô*Ð*à
×ÒØØØØfð	 ñ ô ð ð ×Ò¨ÐÑ/Ô/Ð/à
×ÒØØØAð ñ ô ð ð
 ×Ò˜uÐÑ%Ô%Ð%à
×ÒØØØØ0Ð0Ð0Ø!ð ñ ô ð ð ×ÒØØÝØØ`ð ñ ô ð ð ×ÒØØØwð ñ ô ð ð #'¨"¢* *ˆ6×ÒÑÔÐ°&×2CÒ2CÀDÑ2IÔ2I€Dð
 Œ>Ð-Ð-Ð-°$´.ÀFÒ2JÐ2JÈtÔOfÐjoÒOoÐOoð 	ˆàð 	„Nð
 €Kr   c                ó  — t          | ¦  «        }t          |j        ¦  «         t                               d|› �¦  «         t          ¦   «         }t          |d|j        dk    ¦  «         ||_        t          |d|j	        dk    rdnd|› �¦  «         t          |dt          j        |j        ¦  «        ¦  «         |j        t          j                             d¦  «        k    }|r|j        n|j        }i }|j        st'          ||||¦  «         d S d x}}|j        s9t+          ||||j        rt          j        nt          j        |j        ¬	¦  «        \  }}d
|_        t'          ||||||¬¦  «        }d|_        t'          ||||||¬¦  «         d S )NzArguments: r%   r£   Údevice_namer`   zcuda:r*   rœ   r]   F)rY   r   T)r¯   r   r¢   ry   rz   r   Úsetattrr­   r~   rr   rk   r*   r±   Útorch_model_directoryr¨   r©   rª   Ú
model_namer)   r–   r{   r   r%   rl   rm   r   )r—   r   r~   rV   rU   rX   r   Úllamas           r   Úmainr¶   +  s±  € Ý�D‰>Œ>€DÝ�”ÑÔÐÝ
‡K‚KÐ$˜dÐ$Ð$Ñ%Ô%Ð%Ý‰:Œ:€Dõ ˆD�*˜dœn°Ò6Ñ7Ô7Ð7Ø€D„IÝˆD�-¨$Ô*AÀUÒ*JÐ*J  ÐP^ÐX\ÐP^ÐP^Ñ_Ô_Ð_ÝˆD�(�EœL¨Ô)9Ñ:Ô:Ñ;Ô;Ð;ØÔ/µ2´7·<²<ÀÑ3DÔ3DÒD€NØ"0ÐPˆtŒˆ°dÔ6P€HàÐØŒ;ð nÝ�d˜H nÐ6HÑIÔIÐIÐIÐIàÐˆ�ØŒ~ð 	Ý-ØØØØ.2¬mÐN�Uœ]˜]ÅÄØ”{ðñ ô ‰MˆF�Eð !ˆÔÝ*Ø�(˜NÐ,>ÈeÐ\bð
ñ 
ô 
Ðð
  ˆÔÝ�d˜H nÐ6HÐX]ÐflÐmÑmÔmÐmÐmÐmr   Ú__main__r!   )r   r   r   r   )NN)r   r   rU   rE   rV   rW   rX   rI   rY   rZ   r   r[   )r—   r˜   ).Ú
__future__r   r¥   Úloggingr¨   ru   rx   rK   Úpackaging.versionÚversionrn   rk   rp   Úbenchmark_helperr   Údist_settingsr   r   Úllama_inputsr   r   r	   r
   r   r   Úllama_torchr   Ú(models.torch_export_patches.cache_helperr   r   Útransformers.cache_utilsr   Úonnxruntimer   Ú	getLoggerry   r   r.   r2   r–   r¯   r¶   Ú__name__ÚseedÚrandomÚmanual_seedr8   r   r   ú<module>rÈ      sE  ðð #Ð "Ð "Ð "Ð "Ð "à €€€Ø €€€Ø 	€	€	€	Ø €€€à Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø €€€Ø Ð Ð Ð Ø )Ð )Ð )Ð )Ð )Ð )Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð *Ð )Ð )Ð )Ð )Ð )Ø GÐ GÐ GÐ GÐ GÐ GØ #Ð #Ð #Ð #Ð #Ð #Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1à Ð Ð Ð à	ˆÔ	˜2Ñ	Ô	€ðKð Kð Kð Kð ð  ð  ð  ðFXð Xð Xð6 -1Ø $ð`ð `ð `ð `ð `ðFað að að aðH ð $nð $nð $nð $nð $nðN ˆzÒÐØ€DØ„I‡N‚N�4ÑÔÐØ€EÔ�dÑÔÐØ€D�F„F€F€F€Fð	 Ðr   