§
    û¸¸j5:  ã                   ó.  — d Z ddlZddlZddlZddlZddlZddlmZ ddlZddl	Z	ddl
Z
ddl	mZ d„ Zdej        fd„Zd"ded	edz  fd
„Zdej        dedefd„Zdej        dedefd„Zdej        dedej        fd„Zdede	j        defd„Zdej        dededededefd„Zdej        dedededef
d„Z e	j        ¦   «         ded	edz  dededef
d„¦   «         Zd „ Zed!k    r4 e¦   «         Z  ee j!        e j"        e j#        e j$        e j%        ¦  «         dS dS )#z
Export LLM to onnx
é    N)ÚPath)Únnc                  óv  — d„ t           j        j        _        d„ t           j        j        _        d„ t           j        j        _        d„ t           j        j        _        d„ t           j        j        _        d„ t           j        j        _        d„ t           j        j        _	        d„ t           j        j        _
        d	S )
z1do not init model twice as it slow initializationc                 ó   — | S ©N© ©ÚxÚargsÚkwargss      ú}/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/onnxruntime/transformers/large_model_exporter.pyú<lambda>z*disable_huggingface_init.<locals>.<lambda>   s   € À€ ó    c                 ó   — | S r   r   r	   s      r   r   z*disable_huggingface_init.<locals>.<lambda>   s   € ¸€ r   c                 ó   — | S r   r   r	   s      r   r   z*disable_huggingface_init.<locals>.<lambda>   s   € °q€ r   c                 ó   — | S r   r   r	   s      r   r   z*disable_huggingface_init.<locals>.<lambda>   s   € ¸€ r   c                 ó   — | S r   r   r	   s      r   r   z*disable_huggingface_init.<locals>.<lambda>   ó   € ¸q€ r   c                 ó   — | S r   r   r	   s      r   r   z*disable_huggingface_init.<locals>.<lambda>   s   € ¸a€ r   c                 ó   — | S r   r   r	   s      r   r   z*disable_huggingface_init.<locals>.<lambda>    r   r   c                 ó   — | S r   r   r	   s      r   r   z*disable_huggingface_init.<locals>.<lambda>!   s   € ¸1€ r   N)Útorchr   ÚinitÚkaiming_uniform_Úuniform_Únormal_Ú	constant_Úxavier_uniform_Úxavier_normal_Úkaiming_normal_Úorthogonal_r   r   r   Údisable_huggingface_initr"      sƒ   € ð &BÐ%A…E„H„MÔ"Ø9Ð9…E„H„MÔØ8Ð8…E„H„MÔØ:Ð:…E„H„MÔØ$@Ð$@…E„H„MÔ!Ø#?Ð#?…E„H„MÔ Ø$@Ð$@…E„H„MÔ!Ø <Ð <…E„H„MÔÐÐr   Úmodelc                 ó”  — d}d}|                       ¦   «         D ]E}||                     ¦   «         |                     ¦   «         z  z  }||                     ¦   «         z  }ŒFd}d}|                      ¦   «         D ]E}||                     ¦   «         |                     ¦   «         z  z  }||                     ¦   «         z  }ŒF||z   dz  dz  }|S )z-to calculate how much memory this model needsr   é   )Ú
parametersÚnelementÚelement_sizeÚbuffers)r#   Ú
param_sizeÚ	param_sumÚparamÚbuffer_sizeÚ
buffer_sumÚbufferÚall_sizes           r   Úget_model_parameter_sizer1   $   s×   € à€JØ€IØ×!Ò!Ñ#Ô#ð &ð &ˆØ�e—n’nÑ&Ô&¨×);Ò);Ñ)=Ô)=Ñ=Ñ=ˆ
Ø�U—^’^Ñ%Ô%Ñ%ˆ	ˆ	Ø€KØ€JØ—-’-‘/”/ð (ð (ˆØ�v—’Ñ(Ô(¨6×+>Ò+>Ñ+@Ô+@Ñ@Ñ@ˆØ�f—o’oÑ'Ô'Ñ'ˆ
ˆ
Ø˜[Ñ(¨DÑ0°4Ñ7€HØ€Or   Úhf_modelÚ	cache_dirc                 ó  — t          ¦   «          t          j                             | t          j        |d¬¦  «        }|€| }t          j                             |¦  «        }t           |dd¬¦  «                             ¦   «         ¦  «        }||fS )zU
    get the pretrained torch model from hugginface,
    and sample model-inputs
    T)Útorch_dtyper3   Útrust_remote_codeNzHello, my dog is cuteÚpt)Úreturn_tensors)	r"   ÚtransformersÚAutoModelForCausalLMÚfrom_pretrainedr   Úfloat16ÚAutoTokenizerÚtupleÚvalues)r2   r3   Ú	tokenizerr#   Úsample_inputss        r   Ú"initialize_model_and_sample_inputsrB   4   s�   € õ ÑÔÐåÔ-×=Ò=Ø�eœm°yÐTXð >ñ ô €Eð ÐØˆ	ÝÔ*×:Ò:¸9ÑEÔE€Iå˜)˜)Ð$;ÈDÐQÑQÔQ×XÒXÑZÔZÑ[Ô[€MØ�-ÐÐr   ÚgpulistrA   c                 óx  ‡— d„ }ˆfd„Š|                       ¦   «         } g }|                     |                      |d¬¦  «        ¦  «         t          t	          |                      ¦   «         ¦  «        ¦  «        d         }|                      ¦   «         D �]â\  }}|                     ¦   «         D �]^\  }}	|                     |	                     |d¬¦  «        ¦  «         t          |	¦  «        t          j        j	        fv rÔt          j        t          |	¦  «        t          |¦  «        z  ¦  «        }
t          |	¦  «        D ]‘\  }}|                     |                     |d¬¦  «        ¦  «         |t          ||
z  t          |¦  «        ¦  «                 }|                     |¦  «          ‰||¦  «         t!          d|› d|› d|› d|› �¦  «         Œ’�Œ%|	                     |d         ¦  «         t!          d|› d|› d|d         › �¦  «         �Œ`t          t#          |                     ¦   «         ¦  «        ¦  «        dk    r6|                     |d         ¦  «         t!          d|› d|d         › �¦  «         �Œät          j        ¦   «         5   | |d         |d	         ¬
¦  «         ddd¦  «         n# 1 swxY w Y   | S )z/Make the model executable across multiple GPUs.c           	      ó8  — g }d }|D �]µ}t          |¦  «        t          j        ur|                     |¦  «         �nqt	          | d¦  «        r4|                     |                     | j        j        ¦  «        ¦  «         �n-t	          | d¦  «        rPt          |  	                    ¦   «         |¦  «        j        }|                     |                     |¦  «        ¦  «         nÍt	          t          |  
                    ¦   «         d ¦  «        d¦  «        rR|                     |                     t          |  
                    ¦   «         ¦  «        j        j        ¦  «        ¦  «         nK|�4|j        |k    r)|                     |                     |¦  «        ¦  «         n|                     |¦  «         |€|d         j        }�Œ·|                     ¦   «         D ]8\  }}t          |¦  «        t          j        u r|                     |¦  «        ||<   Œ9t          |¦  «        |fS )NÚweightr&   r   )Útyper   ÚTensorÚappendÚhasattrÚtorF   ÚdeviceÚnextr&   ÚchildrenÚitemsr>   )	ÚmodÚinputsr   Úmodifyed_inputsÚ	first_devÚlayer_inputrL   ÚkeyÚvalues	            r   Úinput_gpu_device_hookz5auto_pipeline_parallel.<locals>.input_gpu_device_hookJ   sä  € ØˆØˆ	Ø!ð 	6ñ 	6ˆKÝ�KÑ Ô ­¬Ð4Ð4Ø×&Ò& {Ñ3Ô3Ð3Ñ3Ý˜˜hÑ'Ô'ð 
4Ø×&Ò& {§~¢~°c´jÔ6GÑ'HÔ'HÑIÔIÐIÑIÝ˜˜lÑ+Ô+ð 4Ý˜cŸnšnÑ.Ô.°Ñ<Ô<ÔC�Ø×&Ò& {§~¢~°fÑ'=Ô'=Ñ>Ô>Ð>Ð>Ý�˜cŸlšl™nœn¨dÑ3Ô3°XÑ>Ô>ð 4Ø×&Ò& {§~¢~µd¸3¿<º<¹>¼>Ñ6JÔ6JÔ6QÔ6XÑ'YÔ'YÑZÔZÐZÐZØÐ&¨;Ô+=ÀÒ+JÐ+JØ×&Ò& {§~¢~°iÑ'@Ô'@ÑAÔAÐAÐAà×&Ò& {Ñ3Ô3Ð3ØÐ Ø+¨AÔ.Ô5�	ùØ Ÿ,š,™.œ.ð 	2ð 	2‰JˆC�Ý�E‰{Œ{�eœlÐ*Ð*Ø#Ÿhšh yÑ1Ô1��s‘øå�oÑ&Ô&¨Ð/Ð/r   c                 ó„   •— |                       |¦  «         |                      ¦   «         D ]} ‰|d         |¦  «         Œd S )Né   )rK   Únamed_children)rP   ÚdevÚlayerÚmove_layer_to_device_rurcs      €r   r]   z9auto_pipeline_parallel.<locals>.move_layer_to_device_rurcc   sQ   ø€ Ø�Šˆs‰ŒˆØ×'Ò'Ñ)Ô)ð 	5ð 	5ˆEØ%Ð% e¨A¤h°Ñ4Ô4Ð4Ð4ð	5ð 	5r   T©Úwith_kwargsr   zmove ú.z to rY   ©Úattention_maskN)ÚhalfrI   Úregister_forward_pre_hookrM   ÚiterrZ   rG   r   r   Ú
ModuleListÚmathÚfloorÚlenÚ	enumerateÚminrK   ÚprintÚlistÚno_grad)r#   rC   rA   rW   Ú	all_hooksÚpre_fixÚtop_nameÚ
top_moduleÚnameÚmoduleÚnum_layers_on_each_gpuÚidxÚ
attn_layerÚto_devr]   s                 @r   Úauto_pipeline_parallelry   G   s2  ø€ ð0ð 0ð 0ð25ð 5ð 5ð 5ð 5ð
 �JŠJ‰LŒL€EØ€IØ×Ò�U×4Ò4Ð5JÐX\Ð4Ñ]Ô]Ñ^Ô^Ð^Ý•4˜×,Ò,Ñ.Ô.Ñ/Ô/Ñ0Ô0°Ô3€GØ %× 4Ò 4Ñ 6Ô 6ð 6ñ 6Ñˆ�*Ø&×5Ò5Ñ7Ô7ð 	@ñ 	@‰LˆD�&Ø×Ò˜V×=Ò=Ð>SÐaeÐ=ÑfÔfÑgÔgÐgÝ�F‰|Œ|¥¤Ô 3Ð4Ð4Ð4Ý)-¬µC¸±K´KÅ#ÀgÁ,Ä,Ñ4NÑ)OÔ)OÐ&Ý'0°Ñ'8Ô'8ð Fð F‘O�C˜Ø×$Ò$ Z×%IÒ%IÐJ_ÐmqÐ%IÑ%rÔ%rÑsÔsÐsà$¥S¨Ð0FÑ)FÍÈGÉÌÑ%UÔ%UÔV�FØ—M’M &Ñ)Ô)Ð)Ø-Ð-¨j¸&ÑAÔAÐAÝÐD 'ÐDÐD¨DÐDÐD°3ÐDÐD¸FÐDÐDÑEÔEÐEÐEñFð —	’	˜' !œ*Ñ%Ô%Ð%ÝÐ>˜gÐ>Ð>¨Ð>Ð>°'¸!´*Ð>Ð>Ñ?Ô?Ð?Ñ?Ý�t�J×-Ò-Ñ/Ô/Ñ0Ô0Ñ1Ô1°QÒ6Ð6Ø�MŠM˜' !œ*Ñ%Ô%Ð%ÝÐ4˜(Ð4Ð4¨°¬
Ð4Ð4Ñ5Ô5Ð5ùå	Œ‰Œð Að AØˆˆm˜AÔ¨}¸QÔ/?Ð@Ñ@Ô@Ð@ðAð Að Añ Aô Að Að Að Að Að Að Aøøøð Að Að Að Aà€Ls   Ê	J/Ê/J3Ê6J3Ú	with_pastc                 ó2  ‡‡— g Šˆfd„}|                       |d¬¦  «        }t          j        | j        ¦  «        j        Št          ‰                     ¦   «         ¦  «        }ˆfd„|D ¦   «         } | |d         |d         ¬¦  «        }|                     ¦   «          ‰d         Š|}t          ‰d         ¦  «        D ]\  }	}
‰d         |	         ||	<   Œ‰d          	                    ¦   «         D ]\  }}| 
                    |¦  «        }	|||	<   Œ t          t          ||d¬	¦  «        ¦  «        D ]d\  }	\  }}t          |¦  «        t          j        u r|                     | j        ¦  «         d
|v r#|||	<   |r | |d         |d         |¬¦  «        n|}Œe|||j        fS )zn
    auto retrieve onnx inputs from torch model as we can't enumlate all possibilities
    for all models
    c                 óB   •— ‰                      ||f¦  «         ‰d         S )Nr   )rI   )Ú_rQ   r   Úuser_inputss      €r   Úhook_for_inputsz-retrieve_onnx_inputs.<locals>.hook_for_inputs‹   s&   ø€ Ø×Ò˜F FÐ+Ñ,Ô,Ð,Ø˜1Œ~Ðr   Tr^   c                 óD   •— g | ]}‰                      |¦  «        j        ‘ŒS r   )ÚgetÚdefault)Ú.0rU   Úforward_paramss     €r   ú
<listcomp>z(retrieve_onnx_inputs.<locals>.<listcomp>“   s*   ø€ ÐLÐLÐL¸#�n×(Ò(¨Ñ-Ô-Ô5ÐLÐLÐLr   r   rY   ra   F)ÚstrictÚ	use_cache)rb   r‡   )rd   ÚinspectÚ	signatureÚforwardr&   rm   ÚkeysÚremoverj   rO   ÚindexÚziprG   r   rH   rK   rL   Úpast_key_values)r#   rA   rz   r   Úhook_handleÚ
input_keysÚdefault_valuesÚoutÚonnx_inputsrv   Ú_valrU   rV   r„   r~   s                @@r   Úretrieve_onnx_inputsr–   „   så  øø€ ð
 €Kðð ð ð ð ð ×1Ò1°/ÈtÐ1ÑTÔT€KåÔ& u¤}Ñ5Ô5Ô@€NÝ�n×)Ò)Ñ+Ô+Ñ,Ô,€JØLÐLÐLÐLÀÐLÑLÔL€NØ
ˆ%�˜aÔ °¸qÔ1AÐ
BÑ
BÔ
B€CØ×ÒÑÔÐØ˜a”.€KØ €KÝ˜{¨1œ~Ñ.Ô.ð /ð /‰	ˆˆTØ& qœ>¨#Ô.ˆ�CÑÐØ! !”n×*Ò*Ñ,Ô,ð !ð !‰
ˆˆUØ×Ò˜sÑ#Ô#ˆØ ˆ�CÑÐÝ&¥s¨:°{È5Ð'QÑ'QÔ'QÑRÔRð vð vÑˆ‰\ˆc�5Ý�‰;Œ;�%œ,Ð&Ð&Ø�HŠH�U”\Ñ"Ô"Ð"Ø˜#ÐÐØ(ˆK˜ÑØclÐu�%�%˜ aÔ(¸ÀqÔ9IÐU^Ð_Ñ_Ô_Ð_ÐruˆCøà�{ CÔ$7Ð7Ð7r   Úsample_inputs_tpÚreturnc                 óÀ  — t           j                             d¦  «        j        dz  dz  }t	          dt          | ¦  «        dz  › d�¦  «         t	          d|dz  › d�¦  «         t          | ¦  «        |dz  k    r¯d„ t          t           j                             ¦   «         ¦  «        D ¦   «         }t          |¦  «        dk    r1t	          t          |¦  «        › d	�¦  «         t          | ||¦  «        } nkt	          d
¦  «         |  
                    ¦   «                              ¦   «         } n5t	          d¦  «         |                      ¦   «                              ¦   «         } | S )a	  
    According to the model size, we will upload it to
    CPU if has no GPU or enough GPU memory,
    Single GPU if has only one GPU in local or model size is enough to fit one GPU
    Multiple GPU if there is more than one gpu in local and model is too large
    r   r%   zModel_Size = z GBztotal_mem_per_cpu = gÍÌÌÌÌÌÜ?c                 ó6   — g | ]}t          j        |¦  «        ‘ŒS r   )r   rL   )rƒ   Úis     r   r…   z.move_to_appropriate_device.<locals>.<listcomp>³   s    € ÐWÐWÐW°�Uœ\¨!™_œ_ÐWÐWÐWr   rY   zk GPUs are used to export onnx,                    Please set CUDA_VISIBLE_DEVICES to use specific GPU groupz5!!!! convert model to float and export onnx using CPUzExport model on a single GPU)r   ÚcudaÚget_device_propertiesÚtotal_memoryrl   r1   ÚrangeÚdevice_countri   ry   ÚcpuÚfloatrc   )r#   r—   Útotal_mem_per_cpuÚdevice_collections       r   Úmove_to_appropriate_devicer¥   §   se  € õ œ
×8Ò8¸Ñ;Ô;ÔHÈ4ÑOÐRVÑVÐå	Ð
EÕ2°5Ñ9Ô9¸DÑ@Ð
EÐ
EÐ
EÑFÔFÐFÝ	Ð
>Ð!2°TÑ!9Ð
>Ð
>Ð
>Ñ?Ô?Ð?Ý Ñ&Ô&Ð):¸TÑ)AÒAÐAØWÐWµe½E¼J×<SÒ<SÑ<UÔ<UÑ6VÔ6VÐWÑWÔWÐÝÐ Ñ!Ô! AÒ%Ð%ÝÝÐ(Ñ)Ô)ð Nð Nð Nñô ð õ +¨5Ð2CÐEUÑVÔVˆEˆEåÐIÑJÔJÐJØ—I’I‘K”K×%Ò%Ñ'Ô'ˆEˆEåÐ,Ñ-Ô-Ð-Ø—
’
‘”×!Ò!Ñ#Ô#ˆØ€Lr   rL   c                 óÞ   — g }| D ]Z}t          |t          j        ¦  «        r)|                     |                     |¦  «        ¦  «         ŒE|                     |¦  «         Œ[t          |¦  «        S )zmove inputs to device)Ú
isinstancer   rH   rI   rK   r>   )rA   rL   Úsample_inputs_Ú
sample_ints       r   Úadapt_inputs_to_devicerª   Ã   sr   € à€NØ#ð .ð .ˆ
Ý�j¥%¤,Ñ/Ô/ð 	.Ø×!Ò! *§-¢-°Ñ"7Ô"7Ñ8Ô8Ð8Ð8à×!Ò! *Ñ-Ô-Ð-Ð-Ý�Ñ Ô Ð r   r”   Útorch_input_namesr�   Úinput_with_pastc           	      óÊ  ‡‡‡— d}ddi}|� t          |¦  «        }t          j        |d         d         j        ¦  «        ‰d         j        d         k                         ¦   «                              d¦  «        }|                     ¦   «         dk    sJ ‚dd|                     ¦   «         di}|s| j        j	        }t          ˆˆfd„t          t          ‰¦  «        ¦  «        D ¦   «         ¦  «        }	d|	v rd	|	v s
J d
¦   «         ‚d}
dddœdddœdœ}t          |	¦  «        D ]A\  Š}||vr8ˆfd„t          ‰‰                              ¦   «         ¦  «        D ¦   «         }|||<   ŒB|r<t          |¦  «        D ],}|	d|› d�fz  }	|	d|› d�fz  }	|||	d         <   |||	d         <   Œ-|s|r&t          |¦  «        D ]}|
d|› d�fz  }
|
d|› d�fz  }
Œt          ‰¦  «        D ]ˆ\  Š}|r�|dk    r|‰‰<   Œ|d	k    rP‰‰         }t          j        |t          j        |j        d         df|j        |j        ¬¦  «        fd¬¦  «        ‰‰<   Œi|dk    r‰‰         }|dd…dd…f         ‰‰<   Œ‰|	|
|fS )z"fetch onnx inputs and outputs namer   Ú
batch_sizeNéÿÿÿÿrY   Úseq_lenc                 ó`   •— g | ]*}t          ‰|         t          j        ¦  «        ¯"‰|         ‘Œ+S r   )r§   r   rH   )rƒ   r›   r”   r«   s     €€r   r…   z2fetch_onnx_inputs_outputs_name.<locals>.<listcomp>å   s9   ø€ ÐqÐqÐq !Í
ÐS^Ð_`ÔSaÕchÔcoÑHpÔHpÐqÐ	˜1Ô	ÐqÐqÐqr   Ú	input_idsrb   z6input_ids and attention_mask must be existed in inputs)Úlogits)r   rY   )r²   rb   c                 ó    •— i | ]
}|‰› d |› �“ŒS )Ú__unknown_dims__r   )rƒ   r›   rv   s     €r   ú
<dictcomp>z2fetch_onnx_inputs_outputs_name.<locals>.<dictcomp>ò   s)   ø€ ÐbÐbÐb¸q˜A #Ð:Ð:°qÐ:Ð:ÐbÐbÐbr   zpast_key_values.z.keyz.valueéþÿÿÿzpresent.r�   )rL   Údtype)Údim)ri   r   ÚtensorÚshapeÚnonzeroÚviewÚnumelÚitemÚconfigÚnum_hidden_layersr>   rŸ   rj   r¹   ÚcatÚonesrL   r¸   )r#   r”   r«   r�   rz   r¬   Únum_of_past_keyÚkv_cache_axisÚ	seq_indexÚonnx_inp_namesÚonnx_out_namesÚonnx_dynamic_axesrs   Úunknown_dimsr›   Ú	attn_maskr²   rv   s    ``              @r   Úfetch_onnx_inputs_outputs_namerÌ   Î   sa  øøø€ ð €OØ˜Ð%€MàÐ"Ý˜oÑ.Ô.ˆÝ”\ /°!Ô"4°QÔ"7Ô"=Ñ>Ô>À+ÈaÄ.ÔBVÐWYÔBZÒZ×cÒcÑeÔe×jÒjÐkmÑnÔnˆ	Ø�ŠÑ Ô  AÒ%Ð%Ð%Ð%Ø˜L¨)¯.ª.Ñ*:Ô*:¸IÐFˆàð 9Øœ,Ô8ˆõ ØqÐqÐqÐqÐq¥u­SÐ1BÑ-CÔ-CÑ'DÔ'DÐqÑqÔqñô €Nð ˜.Ð(Ð(Ð-=ÀÐ-OÐ-OÐ-OØ@ñ .PÔ-OÐOð !€Nà%¨)Ð4Ð4Ø*¨yÐ9Ð9ðð Ðõ
 ˜~Ñ.Ô.ð 3ð 3‰	ˆˆTØÐ(Ð(Ð(ØbÐbÐbÐbÅEÈ+ÐVYÔJZ×J^ÒJ^ÑJ`ÔJ`ÑDaÔDaÐbÑbÔbˆLØ&2Ð˜dÑ#øØð BÝ�Ñ'Ô'ð 	Bð 	BˆAØÐ9°!Ð9Ð9Ð9Ð;Ñ;ˆNØÐ;°!Ð;Ð;Ð;Ð=Ñ=ˆNà4AÐ˜n¨RÔ0Ñ1Ø4AÐ˜n¨RÔ0Ñ1Ð1àð 6�Oð 6Ý�Ñ'Ô'ð 	6ð 	6ˆAØÐ1¨!Ð1Ð1Ð1Ð3Ñ3ˆNØÐ3¨!Ð3Ð3Ð3Ð5Ñ5ˆNˆNåÐ0Ñ1Ô1ð 5ð 5‰	ˆˆTØð 	5ØÐ(Ò(Ð(Ø#2�˜CÑ Ð ØÐ)Ò)Ð)Ø'¨Ô,�	Ý#(¤9Ø¥¤
¨I¬O¸AÔ,>ÀÐ+BÈ9ÔK[ÐclÔcrÐ sÑ sÔ sÐtØð$ñ $ô $�˜CÑ Ð ð ˜Ò$Ð$Ø'¨Ô,�	Ø#,¨Q¨Q¨Q°°°¨VÔ#4�˜CÑ øà˜>Ð+<Ð<Ð<r   Úonnx_io_tupleÚ	onnx_pathÚopsetc                 ól  — |j         }|\  }}}t          j        ¦   «         5 }	t          j                             |	d¦  «        }
t          j                             | t          |¦  «        |
d||||¬¦  «         | 
                    d¬¦  «         |j        |› d�z   
                    d¬¦  «         t          j        t          |
¦  «        ¦  «        }t          j        |t          |¦  «        t          t          j        |	¦  «        ¦  «        dk    d|› d�dd¬	¦  «         d
d
d
¦  «         d
S # 1 swxY w Y   d
S )z do export with torch.onnx.exportztmp.onnxF)r#   r   ÚfÚverboseÚopset_versionÚinput_namesÚoutput_namesÚdynamic_axesT)Ú
missing_okz	_ext.datarY   r%   )Úsave_as_external_dataÚall_tensors_to_one_fileÚlocationÚsize_thresholdÚconvert_attributeN)rs   ÚtempfileÚTemporaryDirectoryÚosÚpathÚjoinr   ÚonnxÚexportr>   ÚunlinkÚparentÚloadÚstrÚ
save_modelri   Úlistdir)r#   rÍ   r”   rÎ   rÏ   Úonnx_model_namerÇ   rÈ   rÉ   Ú
tmpdirnameÚtmp_onnxÚ
onnx_models               r   Údo_export_internalrî     s†  € à”n€OØ8EÑ5€N�NÐ$5õ 
Ô	$Ñ	&Ô	&ð 
¨*Ý”7—<’< 
¨JÑ7Ô7ˆåŒ
×ÒØÝ�{Ñ#Ô#ØØØØ&Ø'Ø*ð 	ñ 		
ô 		
ð 		
ð 	×Ò DÐÑ)Ô)Ð)Ø	Ô	˜Ð9Ð9Ð9Ñ	9×AÒAÈTÐAÑRÔRÐRå”Y�s 8™}œ}Ñ-Ô-ˆ
ÝŒØÝ�	‰NŒNÝ#&¥r¤z°*Ñ'=Ô'=Ñ#>Ô#>ÀÒ#BØ$(Ø'Ð2Ð2Ð2ØØ#ð	
ñ 	
ô 	
ð 	
ð%
ð 
ð 
ñ 
ô 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
øøøð 
ð 
ð 
ð 
ð 
ð 
s   ¡C;D)Ä)D-Ä0D-Úonnx_path_strc                 ó   — t          | |¦  «        \  }}t          ||¦  «        }t          |t          |                     ¦   «         ¦  «        j        ¦  «        }t          |||¦  «        \  }}	}
t          ||	||
|d¦  «        }d}t          |¦  «         	                    ¦   «         }|j
        dk    r||z  }t          |||	||¦  «         |sdS t          ||	||
|d¦  «        }d}|j        |z  }t          |||	||¦  «         dS )z„
    do export
    model: torch model
    onnx_path: where the onnx model saved to
    sample_inputs_tp: inputs for torch model
    Fz
model.onnxz.onnxNTzmodel_with_past.onnx)rB   r¥   rª   rM   r&   rL   r–   rÌ   r   ÚabsoluteÚsuffixrî   rå   )r2   r3   rï   rz   rÏ   r#   r—   rA   r‘   r”   Úpast_key_valuerÍ   rê   rÎ   s                 r   Úexport_onnxrô   6  s(  € õ AÀÈ9ÑUÔUÑ€EÐå& uÐ.>Ñ?Ô?€Eå*Ð+;½TÀ%×BRÒBRÑBTÔBTÑ=UÔ=UÔ=\Ñ]Ô]€Mõ /CÀ5È-ÐYbÑ.cÔ.cÑ+€J�˜^å2°5¸+ÀzÐSaÐclÐnsÑtÔt€Mà"€OÝ˜=Ñ)Ô)×2Ò2Ñ4Ô4€IØÔ˜7Ò"Ð"Ø Ñ/ˆ	å�u˜m¨[¸)ÀUÑKÔKÐKØð Øˆå2°5¸+ÀzÐSaÐclÐnrÑsÔs€Mà,€OØÔ  ?Ñ2€Iå�u˜m¨[¸)ÀUÑKÔKÐKÐKÐKr   c                  ó€  — t          j        ¦   «         } |                      dddt          dgd¬¦  «         |                      ddd	t          d
d¬¦  «         |                      dd	t          dd¬¦  «         |                      ddd	d¬¦  «         |                      dd	t          dd¬¦  «         |                      ¦   «         S )zarguments parsing.z-mz--modelTzmeta-llama/Llama-2-70b-hfz+Pre-trained models in huggingface model hub)ÚrequiredrG   r‚   Úhelpz-sz--saved_pathFz./onnx_models/z"where the onnx model will be savedz--cache_dirNz[cache directly of huggingface, by setting this to avoid useless downloading if you have onez--with_pastÚ
store_truez;The tool will export onnx without past-key-value by default)Úactionr‚   r÷   z--opseté   zothe opset to save onnx model,               try to increase it if this opset doens't have new features you want)ÚargparseÚArgumentParserÚadd_argumentrç   ÚintÚ
parse_args)Úparsers    r   Úparse_argumentsr  Z  s  € åÔ$Ñ&Ô&€Fà
×ÒØØØÝØ,Ð-Ø:ð ñ ô ð ð ×ÒØØØÝØ Ø1ð ñ ô ð ð ×ÒØØÝØØkð ñ ô ð ð ×ÒØØØØKð	 ñ ô ð ð ×ÒØØÝØðSð ñ 	ô 	ð 	ð ×ÒÑÔÐr   Ú__main__r   )&Ú__doc__rû   rˆ   rg   rß   rÝ   Úpathlibr   râ   r   r9   r   r"   ÚModuler1   rç   rB   rm   r>   ry   Úboolr–   r¥   rL   rª   rÌ   rþ   rî   rn   rô   r  Ú__name__r   r#   r3   Ú
saved_pathrz   rÏ   r   r   r   ú<module>r	     sÑ  ððð ð €€€Ø €€€Ø €€€Ø 	€	€	€	Ø €€€Ø Ð Ð Ð Ð Ð à €€€Ø €€€Ø Ð Ð Ð Ø Ð Ð Ð Ð Ð ð
=ð 
=ð 
=ð B¤Ið ð ð ð ð  ð  °ð  ÀÀtÁð  ð  ð  ð  ð&: "¤)ð :°dð :È5ð :ð :ð :ð :ðz 8 ¤	ð  8¸%ð  8ÈDð  8ð  8ð  8ð  8ðF b¤ið À5ð ÈRÌYð ð ð ð ð8!¨%ð !¸¼ð !È%ð !ð !ð !ð !ðA=ØŒ9ðA=àðA=ð ðA=ð ð	A=ð
 ðA=ð ðA=ð A=ð A=ð A=ðH!
˜bœið !
¸ð !
ÈEð !
Ð^bð !
Ðknð !
ð !
ð !
ð !
ðH €„�„ð L˜#ð  L¨#°©*ð  LÀSð  LÐUYð  LÐbeð  Lð  Lð  Lñ „ð LðF+ð +ð +ð\ ˆzÒÐØˆ?ÑÔ€Dà€K�”
˜DœN¨D¬O¸T¼^ÈTÌZÑXÔXÐXÐXÐXð Ðr   