§
    û¸¸j“/  ã                   óÔ   — d dl Z d dlZd dlmZ d dlZd dlmZ d dlmZ d dl	m
Z
 d dlmZmZ d dlmZmZ d dlmZmZ d d	lmZ  e j        e¦  «        Zg d
¢Zg d¢Z G d„ d¦  «        ZdS )é    N)ÚPath)Úfloat_to_float16_max_diff)Ú	OnnxModel)Úoptimize_model)Ú	T5DecoderÚT5DecoderHelper)ÚT5EncoderDecoderInitÚT5EncoderDecoderInitHelper)ÚMT5ForConditionalGenerationÚT5ForConditionalGeneration)ÚInferenceSession)zt5-smallzt5-basezt5-largezt5-3bzt5-11b)zgoogle/mt5-smallzgoogle/mt5-basezgoogle/mt5-largezgoogle/mt5-xlzgoogle/mt5-xxlc                   óª  — e Zd Ze	 	 d(dededededef
d„¦   «         Ze	 	 	 d)ded
edej        dededede	ee
ez  f         fd„¦   «         Ze	 	 	 	 d*dee
z  dej        dededededefd„¦   «         Ze	 	 	 d+dedee         dz  dedefd„¦   «         Ze	 	 	 	 d,dededed ed!eded"ed#ed$efd%„¦   «         Zedee
z  d&edej        defd'„¦   «         ZdS )-ÚT5HelperÚ FÚ
output_dirÚmodel_name_or_pathÚsuffixÚ
new_folderÚreturnc                 óH  — |}t           j                             |¦  «        rt          |¦  «        j        d         }n|                     d¦  «        d          ||z  }|r t           j                             | |¦  «        n| }t           j                             ||dz   ¦  «        S )aÊ  Build onnx path

        Args:
            output_dir (str): output directory
            model_name_or_path (str): pretrained model name, or path to the model checkpoint
            suffix (str, optional): suffix like "_encoder" or "_decoder_fp16" will be appended to file name. Defaults to None.
            new_folder (bool, optional): create a new directory for the model. Defaults to False.

        Returns:
            str: path of onnx model
        éÿÿÿÿú/z.onnx)ÚosÚpathÚisdirr   ÚpartsÚsplitÚjoin)r   r   r   r   Ú
model_nameÚ	directorys         ú|/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/onnxruntime/transformers/models/t5/t5_helper.pyÚget_onnx_pathzT5Helper.get_onnx_path!   s•   € ð$ (ˆ
ÝŒ7�=Š=Ð+Ñ,Ô,ð 	&ÝÐ0Ñ1Ô1Ô7¸Ô;ˆJˆJà×Ò˜SÑ!Ô! "Ô%Ð%à�fÑˆ
à<FÐV•B”G—L’L ¨ZÑ8Ô8Ð8ÈJˆ	ÝŒw�|Š|˜I z°GÑ';Ñ<Ô<Ð<ó    Út5Ú	cache_dirÚdeviceÚ
model_typeÚstate_dict_pathÚencoder_decoder_initc                 óà  — |dk    rt          j        | |¬¦  «        }n,|dk    rt          j        | |¬¦  «        }nt          d¦  «        ‚|r'|                     t          j        |¦  «        ¦  «         t          |j        |j	        |j
        ¦  «        }|                     ¦   «                              |¦  «         t          |j        |j        |j	        |j
        d| ¬¦  «        }|rdnd}	|	|d	|iS )
a{  Load model given a pretrained name or path, then build models for ONNX conversion.

        Args:
            model_name_or_path (str): pretrained model name or path
            cache_dir (str): cache directory
            device (torch.device): device to run the model
            model_type (str, optional): model type "t5" or "mt5"
            state_dict_path(str, optional): state dictionary path
            encoder_decoder_init (bool, optional): combine encoder and decoder kv cache initialization into one model.
        Returns:
            Dict[str, torch.nn.Module]: mapping from name to modules for ONNX conversion.
        r$   )r%   Úmt5z only support mode_type=t5 or mt5N)Údecoder_start_token_idÚoutput_cross_onlyr)   ÚencoderÚdecoder)r   Úfrom_pretrainedr   Ú
ValueErrorÚload_state_dictÚtorchÚloadr   r/   Úlm_headÚconfigÚevalÚtor	   r.   )
r   r%   r&   r'   r(   r)   Úmodelr/   r.   Úencoder_names
             r!   Ú
load_modelzT5Helper.load_model>   s  € ð* ˜ÒÐÝ.Ô>Ð?QÐ]fÐgÑgÔgˆEˆEØ˜5Ò Ð Ý/Ô?Ð@RÐ^gÐhÑhÔhˆEˆEåÐ?Ñ@Ô@Ð@àð 	?Ø×!Ò!¥%¤*¨_Ñ"=Ô"=Ñ>Ô>Ð>å˜EœM¨5¬=¸%¼,ÑGÔGˆØ�Š‰Œ×Ò˜&Ñ!Ô!Ð!å&ØŒMØŒMØŒMØŒLØ#'Ø"6Ð6ð
ñ 
ô 
ˆð 2FÐTÐ-Ð-È9ˆØ˜g y°'Ð:Ð:r#   Tr9   Úonnx_model_pathÚverboseÚuse_external_data_formatÚuse_decoder_input_idsÚuse_int32_inputsc           	      óš   — t          | t          ¦  «        rt          j        | ||||||¦  «         d S t	          j        | |||||¦  «         d S )N)Ú
isinstancer	   r
   Úexport_onnxr   )r9   r&   r<   r=   r>   r?   r@   s          r!   rC   zT5Helper.export_onnxl   s~   € õ �eÕ1Ñ2Ô2ð 	Ý&Ô2ØØØØ%ØØ(Ø ñô ð ð ð õ Ô'ØØØØØ(Ø ñô ð ð ð r#   NÚ
onnx_modelÚop_block_listÚforce_fp16_logitsÚuse_symbolic_shape_inferc                 óè  ‡‡— ‰€g d¢Šd„ |                       ¦   «         D ¦   «         }t          ‰¦  «        }|                     |¦  «        }t                               d|› d|› �¦  «         |                      ¦   «         j        d         j        }d}|                      ¦   «         }	||	v sJ ‚|	|         }
d}|
j	        dk    r„|
}t                               d	|
j        › �¦  «         d}|
j
        D ]}|                      |¦  «        }|� nŒt          |¦  «        }t                               d
|
j        › d|› �¦  «         |dk     }n*t                               d|
j	        › d|
j        › �¦  «         g }g Š|s|�|s|g}|j        gŠd‰v�r|                      ¦   «         }d}d}|                      d¦  «        }|r d}|D ]™}
|
j        ‰vrŽ|                      |
|	¦  «        }|                      |
|¦  «        }ˆˆfd„|D ¦   «         }ˆˆfd„|D ¦   «         }t'          |¦  «        t'          |¦  «        z   dk    r!‰                     |
j        ¦  «         |dz  }d}Œš|° t'          |¦  «        |z
  }t                               d|› d|› �¦  «         t                               d‰› �¦  «         |‰‰|dœ}t                               d|› �¦  «         |r | j        dddi|¤Ž nddlm}  || j        fddi|¤Ž |S )a¢  Convert model to mixed precision.
           It detects whether original model has fp16 precision weights, and set parameters for float16 conversion automatically.
        Args:
            onnx_model (OnnxModel): optimized ONNX model
            op_block_list (List[str], optional): operators need to run in fp32.
            force_fp16_logits (bool, optional): force logits and last MatMul node to be in float16. Defaults to False.
            use_symbolic_shape_infer (bool, optional): use symbolic shape inference to convert float to float16. Defaults to True.
        Returns:
            parameters(dict): a dictionary of parameters used in float16 conversion
        N)ÚSimplifiedLayerNormalizationÚ SkipSimplifiedLayerNormalizationÚReluÚAddc                 ó   — h | ]	}|j         ’Œ
S © ©Úop_type)Ú.0Únodes     r!   ú	<setcomp>z0T5Helper.auto_mixed_precision.<locals>.<setcomp>£   s   € ÐCÐCÐC¨�t”|ÐCÐCÐCr#   z	fp32 op: z
 fp16 op: r   FÚMatMulz#Found last MatMul node for logits: z3max diff of converting weights in last MatMul node z: g�íµ ÷Æ°>z-Failed to find MatMul node for logits. Found z	 of node rL   Tc                 ó.   •— g | ]}|j         ‰v s|‰v ¯|‘ŒS rN   rO   )rQ   ÚchildÚnode_block_listrE   s     €€r!   ú
<listcomp>z1T5Helper.auto_mixed_precision.<locals>.<listcomp>Ô   s8   ø€ ð ,ð ,ð ,Ø&+¸5¼=ÈMÐ;YÐ;YÐ]bÐfuÐ]uÐ]u˜EÐ]uÐ]uÐ]ur#   c                 ó.   •— g | ]}|j         ‰v s|‰v ¯|‘ŒS rN   rO   )rQ   ÚparentrW   rE   s     €€r!   rX   z1T5Helper.auto_mixed_precision.<locals>.<listcomp>×   s8   ø€ ð +ð +ð +Ø'-¸F¼NÈmÐ<[Ð<[Ð_eÐixÐ_xÐ_x˜FÐ_xÐ_xÐ_xr#   é   z#node counter of Add operator: fp32=z fp16=znode_block_list: )Úkeep_io_typesrE   rW   Úforce_fp16_initializersz!auto_mixed_precision parameters: rG   )Úconvert_float_to_float16Údisable_shape_inferrN   )ÚnodesÚsetÚ
differenceÚloggerÚinfoÚgraphÚoutputÚnameÚoutput_name_to_noderP   ÚinputÚget_initializerr   ÚdebugÚwarningÚinput_name_to_nodesÚget_nodes_by_op_typeÚget_parentsÚget_childrenÚlenÚappendr^   Úfloat16r9   )rD   rE   rF   rG   Úop_full_setÚfp32_op_setÚfp16_op_setÚlogits_output_nameÚis_weight_fp16_precisionrh   rR   Úlast_matmul_nodeÚinitializerri   Úmax_diffr\   rm   Úfp32_addÚchangedÚ	add_nodesÚparentsÚchildrenÚblocked_childrenÚblocked_parentsÚfp16_addÚ
parametersr^   rW   s    `                         @r!   Úauto_mixed_precisionzT5Helper.auto_mixed_precisionŠ   s  øø€ ð" Ð ðð ð ˆMð DÐC°
×0@Ò0@Ñ0BÔ0BÐCÑCÔCˆÝ˜-Ñ(Ô(ˆØ!×,Ò,¨[Ñ9Ô9ˆÝ�ŠÐD ÐDÐD°{ÐDÐDÑEÔEÐEð (×-Ò-Ñ/Ô/Ô6°qÔ9Ô>Ðð $)Ð Ø(×<Ò<Ñ>Ô>ÐØ!Ð%8Ð8Ð8Ð8Ð8Ø"Ð#5Ô6ˆØÐØŒ<˜8Ò#Ð#Ø#ÐÝ�KŠKÐI¸d¼iÐIÐIÑJÔJÐJØˆKØœð ð �Ø(×8Ò8¸Ñ?Ô?�ØÐ*Ø�Eð +õ
 1°Ñ=Ô=ˆHÝ�LŠLÐfÈtÌyÐfÐfÐ\dÐfÐfÑgÔgÐgØ'/°$¢Ð$Ð$å�NŠNÐmÈ4Ì<ÐmÐmÐbfÔbkÐmÐmÑnÔnÐnàˆØˆØ(ð 	6Ð/?Ð/KÐUfÐ/Kà/Ð0ˆMØ/Ô4Ð5ˆOà˜Ð%Ñ%Ø",×"@Ò"@Ñ"BÔ"BÐØˆHØˆGØ"×7Ò7¸Ñ>Ô>ˆIØð +Ø�Ø%ð +ð +�DØ”y¨Ð7Ð7Ø",×"8Ò"8¸Ð?RÑ"SÔ"S˜Ø#-×#:Ò#:¸4ÐATÑ#UÔ#U˜ð,ð ,ð ,ð ,ð ,Ø/7ð,ñ ,ô ,Ð(ð+ð +ð +ð +ð +Ø18ð+ñ +ô +˜õ  Ð 0Ñ1Ô1µC¸Ñ4HÔ4HÑHÈAÒMÐMØ+×2Ò2°4´9Ñ=Ô=Ð=Ø$¨™M˜HØ&*˜Gøð! ð +õ" ˜9‘~”~¨Ñ0ˆHÝ�KŠKÐX¸hÐXÐXÈhÐXÐXÑYÔYÐYå�ŠÐ9¨Ð9Ð9Ñ:Ô:Ð:ð +Ø*Ø.Ø'?ð	
ð 
ˆ
õ 	�ŠÐD¸
ÐDÐDÑEÔEÐEØ#ð 	Ø/ˆJÔ/Ð\Ð\ÈÐ\ÐQ[Ð\Ð\Ð\Ð\ð 9Ð8Ð8Ð8Ð8Ð8à$Ð$ØÔ ðð à$(ðð ðð ð ð Ðr#   Úoptimized_model_pathÚ
is_float16Únum_attention_headsÚhidden_sizer…   Úuse_gpuÚforce_fp16_ioc	           	      ó
  — ddl m}	 d}
|r |	d¦  «        }
| |
_        t          | d||d|
|¬¦  «        }|r5|rt                               ||¬¦  «         n|                     |¬¦  «         |                     ||d¬	¦  «         dS )
zHOptimize ONNX model with an option to convert it to use mixed precision.r   )ÚFusionOptionsNr$   )r'   Ú	num_headsr‰   Ú	opt_levelÚoptimization_optionsrŠ   )rF   )Úcast_input_outputT)Úall_tensors_to_one_file)Úfusion_optionsr�   Úenable_skip_layer_normr   r   r…   Ú convert_model_float32_to_float16Úsave_model_to_file)r<   r†   r‡   rˆ   r‰   r>   r…   rŠ   r‹   r�   r�   Úms               r!   Úoptimize_onnxzT5Helper.optimize_onnxû   sÕ   € ð 	1Ð0Ð0Ð0Ð0Ð0à#ÐØð 	SØ#0 =°Ñ#6Ô#6Ð à>RÐ:RÐ Ô7åØØØ)Ø#ØØ!5Øð
ñ 
ô 
ˆð ð 	TØ#ð TÝ×-Ò-¨aÀ=Ð-ÑQÔQÐQÐQà×2Ò2À]Ð2ÑSÔSÐSà	×ÒÐ1Ð3KÐeiÐÑjÔjÐjÐjÐjr#   Úort_sessionc                 óˆ   — t          | t          ¦  «        rt          j        | |||¦  «        S t	          j        | |||¦  «        S )zQCompare the result from PyTorch and OnnxRuntime to verify the ONNX model is good.)rB   r	   r
   Úverify_onnxr   )r9   r™   r&   r@   s       r!   r›   zT5Helper.verify_onnx#  sH   € õ �eÕ1Ñ2Ô2ð 	hÝ-Ô9¸%ÀÈfÐVfÑgÔgÐgåÔ*¨5°+¸vÐGWÑXÔXÐXr#   )r   F)r$   r   F)TFTF)NFT)FTFF)Ú__name__Ú
__module__Ú__qualname__ÚstaticmethodÚstrÚboolr"   r3   r&   Údictr	   r   r;   rC   r   Úlistr…   Úintr˜   r   r›   rN   r#   r!   r   r       sÄ  € € € € € Øð Ø ð	=ð =Øð=àð=ð ð=ð ð	=ð
 
ð=ð =ð =ñ „\ð=ð8 ð
 Ø!Ø%*ð+;ð +;Øð+;àð+;ð ”ð+;ð ð	+;ð
 ð+;ð #ð+;ð 
ˆcÐ'¨)Ñ3Ð3Ô	4ð+;ð +;ð +;ñ „\ð+;ðZ ð
 Ø).Ø&*Ø!&ðð ØÐ/Ñ/ðà”ðð ðð ð	ð
 #'ðð  $ðð ðð ð ñ „\ðð: ð +/Ø"'Ø)-ð	nð nØðnà˜C”y 4Ñ'ðnð  ðnð #'ð	nð nð nñ „\ðnð` ð */Ø%)ØØ#ð%kð %kØð%kà!ð%kð ð%kð !ð	%kð
 ð%kð #'ð%kð #ð%kð ð%kð ð%kð %kð %kñ „\ð%kðN ð
YØÐ/Ñ/ð
Yà%ð
Yð ”ð
Yð ð	
Yð 
Yð 
Yñ „\ð
Yð 
Yð 
Yr#   r   )Úloggingr   Úpathlibr   r3   rs   r   rD   r   Ú	optimizerr   Ú
t5_decoderr   r   Út5_encoder_decoder_initr	   r
   Útransformersr   r   Úonnxruntimer   Ú	getLoggerrœ   rc   ÚPRETRAINED_T5_MODELSÚPRETRAINED_MT5_MODELSr   rN   r#   r!   ú<module>r¯      s:  ðð €€€Ø 	€	€	€	Ø Ð Ð Ð Ð Ð à €€€Ø -Ð -Ð -Ð -Ð -Ð -Ø  Ð  Ð  Ð  Ð  Ð  Ø $Ð $Ð $Ð $Ð $Ð $Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø TÐ TÐ TÐ TÐ TÐ TÐ TÐ TØ PÐ PÐ PÐ PÐ PÐ PÐ PÐ Pà (Ð (Ð (Ð (Ð (Ð (à	ˆÔ	˜8Ñ	$Ô	$€àMÐMÐMÐ ðð ð Ð ðNYð NYð NYð NYð NYñ NYô NYð NYð NYð NYr#   