§
    û¸¸jÐÁ  ã                   ó–  — d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlZd dl	Z	d dl
Z
d dlmZ dddddd	d
dddœ	ZdddddœZd„ Zd„ ZdBd„Zdededefd„Zdedededefd„Zdededededef
d „Z	 dCded"ed#efd$„Zd%efd&„Z	 dCded"efd'„Z	 dCdeded(ededed)ed*eded+ed,ed-ed#efd.„Z	 	 	 dDdededed0efd1„Z	 	 dEded"efd2„Z	 	 dEdeded(ededed)ed*eded+ed,ed0ed#efd3„Z	 	 dFd4ed5ededed)ed*eded+ed,ed6ed7ed8efd9„Z 	 	 	 dGd4ed5edededed)ed*eded+ed,ed6ed7ed8ed#efd:„Z!	 	 	 dGd4ed5ededed)ed*eded+ed,ed6ed7ed#efd;„Z"	 	 	 dGd4ed5ededed)ed*eded+ed,ed6ed7ed#efd<„Z#	 dHdededededed)ed*eded+ed,ed#efd=„Z$d>„ Z%dHd?„Z&d@„ Z'e(dAk    r4d dl)Z)	  e'¦   «          dS # e*$ r  e)j+         ej,        ¦   «         Ž  Y dS w xY wdS )Ié    N)ÚPath©Úmeasure_memoryzrunwayml/stable-diffusion-v1-5zstabilityai/stable-diffusion-2z stabilityai/stable-diffusion-2-1z+stabilityai/stable-diffusion-xl-refiner-1.0z/stabilityai/stable-diffusion-3-medium-diffusersz'stabilityai/stable-diffusion-3.5-mediumz&stabilityai/stable-diffusion-3.5-largez black-forest-labs/FLUX.1-schnellzblack-forest-labs/FLUX.1-dev)	ú1.5z2.0ú2.1zxl-1.0z3.0Mz3.5Mz3.5LzFlux.1SzFlux.1DÚCUDAExecutionProviderÚROCMExecutionProviderÚMIGraphXExecutionProviderÚTensorrtExecutionProvider)ÚcudaÚrocmÚmigraphxÚtensorrtc                  ó   — g d¢} d}| |fS )N)
z.a photo of an astronaut riding a horse on marsz@cute grey cat with blue eyes, wearing a bowtie, acrylic paintingzia cute magical flying dog, fantasy art drawn by disney concept artists, highly detailed, digital paintingzdan illustration of a house with large barn with many cute flower pots and beautiful blue sky sceneryzgone apple sitting on a table, still life, reflective, full color photograph, centered, close-up productzWbackground texture of stones, masterpiece, artistic, stunning photo, award winner photozSnew international organic style house, tropical surroundings, architecture, 8k, hdrznbeautiful Renaissance Revival Estate, Hobbit-House, detailed painting, warm colors, 8k, trending on Artstationzcblue owl, big green eyes, portrait, intricate metal design, unreal engine, octane render, realisticzldelicate elvish moonstone necklace on a velvet background, symmetrical intricate motifs, leaves, flowers, 8kz*bad composition, ugly, abnormal, malformed© )ÚpromptsÚnegative_prompts     úŠ/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/onnxruntime/transformers/models/stable_diffusion/benchmark.pyÚexample_promptsr   )   s&   € ðð ð €Gð C€Oà�OÐ#Ð#ó    c                  ó   — dS )N)zwarm upÚbadr   r   r   r   Úwarmup_promptsr   <   s   € ØÐr   c                 ó(   — t          d|| |¬¦  «        S )NT)Úis_gpuÚfuncÚmonitor_typeÚstart_memoryr   )r   r   r   s      r   Úmeasure_gpu_memoryr   @   s   € Ý ¨D¸|ÐZfÐgÑgÔgÐgr   Ú
model_nameÚ	directoryÚdisable_safety_checkerc                 ó„  — ddl m}m} dd l}|�Nt          j                             |¦  «        sJ ‚|                     ¦   «         }|                     |||¬¦  «        }n|                     | d|d¬¦  «        }| 	                    |j
        j        ¦  «        |_
        |                     d¬¦  «         |rd |_        d |_        |S )Nr   )ÚDDIMSchedulerÚOnnxStableDiffusionPipeline)ÚproviderÚsess_optionsÚonnxT)Úrevisionr&   Úuse_auth_token©Údisable)Ú	diffusersr$   r%   ÚonnxruntimeÚosÚpathÚexistsÚSessionOptionsÚfrom_pretrainedÚfrom_configÚ	schedulerÚconfigÚset_progress_bar_configÚsafety_checkerÚfeature_extractor)	r    r!   r&   r"   r$   r%   r.   Úsession_optionsÚpipes	            r   Úget_ort_pipeliner<   D   sô   € ØDÐDÐDÐDÐDÐDÐDÐDàÐÐÐàÐÝŒw�~Š~˜iÑ(Ô(Ð(Ð(Ð(Ø%×4Ò4Ñ6Ô6ˆØ*×:Ò:ØØØ(ð ;ñ 
ô 
ˆˆð +×:Ò:ØØØØð	 ;ñ 
ô 
ˆð #×.Ò.¨t¬~Ô/DÑEÔE€D„NØ× Ò ¨Ð Ñ.Ô.Ð.àð &Ø"ˆÔØ!%ˆÔà€Kr   Úenable_torch_compileÚuse_xformersc                 óf  — d| v r„ddl m} |                     | t          j        ¬¦  «                             d¦  «        }|rF|j                             t          j        ¬¦  «         t          j        |j        dd¬	¦  «        |_        |S d
| v r„ddl m	} |                     | t          j        ¬¦  «                             d¦  «        }|rF|j                             t          j        ¬¦  «         t          j        |j        dd¬	¦  «        |_        |S ddl m
}m} ddlm}	m}
 |                     | |
¬¦  «                             d¦  «        }|j                             |	¬¦  «         |r|                     ¦   «          |rit          j        |j        ¦  «        |_        t          j        |j        ¦  «        |_        t          j        |j        ¦  «        |_        t#          d¦  «         |                     |j        j        ¦  «        |_        |                     d¬¦  «         |rd |_        d |_        |S )NÚFLUXr   )ÚFluxPipeline)Útorch_dtyper   )Úmemory_formatzmax-autotuneT)ÚmodeÚ	fullgraphzstable-diffusion-3)ÚStableDiffusion3Pipeline)r$   ÚStableDiffusionPipeline)Úchannels_lastÚfloat16z)Torch compiled unet, vae and text_encoderr+   )r-   rA   r3   ÚtorchÚbfloat16ÚtoÚtransformerrH   ÚcompilerF   r$   rG   rI   ÚunetÚ*enable_xformers_memory_efficient_attentionÚvaeÚtext_encoderÚprintr4   r5   r6   r7   r8   r9   )r    r"   r=   r>   rA   r;   rF   r$   rG   rH   rI   s              r   Úget_torch_pipelinerT   b   sG  € Ø�ÐÐØ*Ð*Ð*Ð*Ð*Ð*à×+Ò+¨JÅEÄNÐ+ÑSÔS×VÒVÐW]Ñ^Ô^ˆØð 	dØÔ×Ò­eÔ.AÐÑBÔBÐBÝ$œ}¨TÔ-=ÀNÐ^bÐcÑcÔcˆDÔØˆà˜zÐ)Ð)Ø6Ð6Ð6Ð6Ð6Ð6à'×7Ò7¸
ÕPUÔP^Ð7Ñ_Ô_×bÒbÐciÑjÔjˆØð 	dØÔ×Ò­eÔ.AÐÑBÔBÐBÝ$œ}¨TÔ-=ÀNÐ^bÐcÑcÔcˆDÔØˆà@Ð@Ð@Ð@Ð@Ð@Ð@Ð@Ø,Ð,Ð,Ð,Ð,Ð,Ð,Ð,à"×2Ò2°:È7Ð2ÑSÔS×VÒVÐW]Ñ^Ô^€Dà„I‡L‚L˜}€LÑ-Ô-Ð-àð :Ø×7Ò7Ñ9Ô9Ð9àð ;Ý”M $¤)Ñ,Ô,ˆŒ	Ý”= ¤Ñ*Ô*ˆŒÝ!œM¨$Ô*;Ñ<Ô<ˆÔÝÐ9Ñ:Ô:Ð:à"×.Ò.¨t¬~Ô/DÑEÔE€D„NØ× Ò ¨Ð Ñ.Ô.Ð.àð &Ø"ˆÔØ!%ˆÔà€Kr   ÚengineÚ
batch_sizeÚstepsc                 óˆ   — |                      d¦  «        d                              dd¦  «        }| › d|› d|› d|› �|rdnd	z   S )
Nú/éÿÿÿÿzstable-diffusion-ÚsdÚ_Ú_bÚ_sÚ Ú_safe)ÚsplitÚreplace)rU   r    rV   rW   r"   Úshort_model_names         r   Úget_image_filename_prefixrd   �   sa   € Ø!×'Ò'¨Ñ,Ô,¨RÔ0×8Ò8Ð9LÈdÑSÔSÐØÐ@Ð@Ð'Ð@Ð@¨:Ð@Ð@¸Ð@Ð@ÐJ`ÐDmÀBÀBÐfmÑnÐnr   FÚimage_filename_prefixÚskip_warmupc                 óÚ  ‡ ‡‡‡‡‡
— ddl m} t          ‰ |¦  «        sJ ‚t          ¦   «         \  }}ˆˆˆ ˆ
ˆˆfd„}t	          |	||¦  «        }t	          |	||¦  «        } |¦   «          g }t          |¦  «        D ]±\  }}||k    r n¥t          j        ¦   «         } ‰ |g‰z  ‰‰‰|g‰z  ¬¦  «        j        }t          j        ¦   «         }||z
  }|                     |¦  «         t          d|d›d�¦  «         t          |¦  «        D ]#\  }}| 
                    |› d|› d|› d	�¦  «         Œ$Œ²dd
lm} d|‰‰‰‰||t          |¦  «        t          |¦  «        z  t          j        |¦  «        ||dœS )Nr   )r%   c                  ób   •— ‰rd S t          ¦   «         \  } } ‰| g‰z  ‰‰‰|g‰z  ¬¦  «         d S )N©ÚpromptÚheightÚwidthÚnum_inference_stepsr   ©r   )rj   ÚnegativerV   rk   r;   rf   rW   rl   s     €€€€€€r   Úwarmupz run_ort_pipeline.<locals>.warmup§   s`   ø€ Øð 	ØˆFÝ)Ñ+Ô+Ñˆ�ØˆØ�8˜jÑ(ØØØ %Ø%˜J¨Ñ3ð	
ñ 	
ô 	
ð 	
ð 	
ð 	
r   ri   úInference took ú.3fú secondsr\   ú.jpg©Ú__version__r.   ©rU   Úversionrk   rl   rW   rV   Úbatch_countÚnum_promptsÚaverage_latencyÚmedian_latencyÚfirst_run_memory_MBÚsecond_run_memory_MB)r-   r%   Ú
isinstancer   r   Ú	enumerateÚtimeÚimagesÚappendrS   Úsaver.   rv   ÚsumÚlenÚ
statisticsÚmedian)r;   rV   re   rk   rl   rW   rz   ry   r   Úmemory_monitor_typerf   r%   r   r   rp   Úfirst_run_memoryÚsecond_run_memoryÚlatency_listÚirj   Úinference_startr‚   Úinference_endÚlatencyÚkÚimageÚort_versions   `` ```    `                r   Úrun_ort_pipeliner”   ”   s  øøøøøø€ ð 6Ð5Ð5Ð5Ð5Ð5å�dÐ7Ñ8Ô8Ð8Ð8Ð8å.Ñ0Ô0Ñ€Gˆ_ð

ð 

ð 

ð 

ð 

ð 

ð 

ð 

ð 

ð 

õ *Ð*=¸vÀ|ÑTÔTÐÝ*Ð+>ÀÈÑUÔUÐà
€F�H„H€Hà€LÝ˜wÑ'Ô'ð ?ð ?‰	ˆˆ6Ø�ÒÐØˆEÝœ)™+œ+ˆØ�Ø�8˜jÑ(ØØØ %Ø,Ð-°
Ñ:ð
ñ 
ô 
ô ð 	õ œ	™œˆØ /Ñ1ˆØ×Ò˜GÑ$Ô$Ð$ÝÐ5 Ð5Ð5Ð5Ð5Ñ6Ô6Ð6Ý! &Ñ)Ô)ð 	?ð 	?‰HˆAˆuØ�JŠJÐ/Ð=Ð=°!Ð=Ð=°aÐ=Ð=Ð=Ñ>Ô>Ð>Ð>ð	?ð 7Ð6Ð6Ð6Ð6Ð6ð  ØØØØØ Ø"Ø"Ý˜|Ñ,Ô,­s°<Ñ/@Ô/@Ñ@Ý$Ô+¨LÑ9Ô9Ø/Ø 1ðð ð r   Úreturnc                 ó¼   — |s|rd| in	d| g|z  ini }t           j                             ¦   «         r+t          j        d¬¦  «                             d¦  «        |d<   |S )Nr   r   )Údeviceé{   Ú	generator)rJ   r   Úis_availableÚ	GeneratorÚmanual_seed)r   Úuse_num_images_per_promptÚis_fluxrV   Úkwargss        r   Úget_negative_prompt_kwargsr    ß   s€   € ð ð	à(ðEÐ Ð0Ð0à# oÐ%6¸Ñ%CÐDÐDð ð õ „z×ÒÑ Ô ð NÝ#œo°VÐ<Ñ<Ô<×HÒHÈÑMÔMˆˆ{Ñà€Mr   c                 óž  ‡ ‡‡‡‡‡
‡— t          ¦   «         \  }}dd l}t          ‰ |j        ¦  «        Šˆˆˆˆ ˆ
ˆˆfd„}t	          |	||¦  «        }t	          |	||¦  «        } |¦   «          t          j        d¦  «         g }t          |¦  «        D ]ø\  }}||k    r nìt
          j         	                    ¦   «          t          j
        ¦   «         }t          |d‰‰¦  «        } ‰ d|g‰z  ‰‰‰dœ|¤Žj        }t
          j         	                    ¦   «          t          j
        ¦   «         }||z
  }|                     |¦  «         t          d|d›d�¦  «         t          |¦  «        D ]#\  }}|                     |› d|› d|› d	�¦  «         Œ$Œùd
t
          j        ‰‰‰‰||t#          |¦  «        t%          |¦  «        z  t'          j        |¦  «        ||dœS )Nr   c                  óx   •— ‰rd S t          ¦   «         \  } }t          |d‰‰¦  «        } ‰d| g‰z  ‰‰	‰dœ|¤Ž d S )NF©rj   rk   rl   rm   r   ©r   r    )
rj   ro   Úextra_kwargsrV   rk   rž   r;   rf   rW   rl   s
      €€€€€€€r   rp   z"run_torch_pipeline.<locals>.warmup  sc   ø€ Øð 	ØˆFÝ)Ñ+Ô+Ñˆ�Ý1°(¸EÀ7ÈJÑWÔWˆØˆÐq�V�H˜zÑ)°&ÀÐ[`ÐqÐqÐdpÐqÐqÐqÐqÐqr   Fr£   rq   rr   rs   r\   rt   rJ   rw   r   )r   r-   r   rA   r   rJ   Úset_grad_enabledr€   r   Úsynchronizer�   r    r‚   rƒ   rS   r„   rv   r…   r†   r‡   rˆ   )r;   rV   re   rk   rl   rW   rz   ry   r   r‰   rf   r   r   r-   rp   rŠ   r‹   rŒ   r�   rj   rŽ   r¥   r‚   r�   r�   r‘   r’   rž   s   `` ```    `                @r   Úrun_torch_pipeliner¨   ò   se  øøøøøøø€ õ  /Ñ0Ô0Ñ€Gˆ_àÐÐÐå˜˜yÔ5Ñ6Ô6€Gðrð rð rð rð rð rð rð rð rð rð rõ *Ð*=¸vÀ|ÑTÔTÐÝ*Ð+>ÀÈÑUÔUÐà
€F�H„H€Hå	Ô˜5Ñ!Ô!Ð!à€LÝ˜wÑ'Ô'ð ?ð ?‰	ˆˆ6Ø�ÒÐØˆEÝŒ
×ÒÑ Ô Ð Ýœ)™+œ+ˆÝ1°/À5È'ÐS]Ñ^Ô^ˆØ�ð 
Ø�8˜jÑ(ØØØ %ð	
ð 
ð
 ð
ð 
ô ð 	õ 	Œ
×ÒÑ Ô Ð Ýœ	™œˆØ /Ñ1ˆØ×Ò˜GÑ$Ô$Ð$ÝÐ5 Ð5Ð5Ð5Ð5Ñ6Ô6Ð6Ý! &Ñ)Ô)ð 	?ð 	?‰HˆAˆuØ�JŠJÐ/Ð=Ð=°!Ð=Ð=°aÐ=Ð=Ð=Ñ>Ô>Ð>Ð>ð	?ð ÝÔ$ØØØØ Ø"Ø"Ý˜|Ñ,Ô,­s°<Ñ/@Ô/@Ñ@Ý$Ô+¨LÑ9Ô9Ø/Ø 1ðð ð r   r&   rk   rl   rz   ry   Útuningc                 óx  — |}|r|dv r|dddœf}t          j         ¦   «         }t          | |||¦  «        }t          j         ¦   «         }t          d||z
  › d�¦  «         t          d| |||¦  «        }t	          ||||||||	|
||¬¦  «        }|                     | ||                     dd	¦  «        |d
dœ¦  «         |S )N)r   r	   é   )Útunable_op_enableÚtunable_op_tuning_enableúModel loading took rs   Úort©rf   ÚExecutionProviderr_   F©r    r!   r&   r"   Úenable_cuda_graph)r�   r<   rS   rd   r”   Úupdaterb   )r    r!   r&   rV   r"   rk   rl   rW   rz   ry   r   r‰   r©   rf   Úprovider_and_optionsÚ
load_startr;   Úload_endre   Úresults                       r   Úrun_ortr¹   ;  s  € ð  $ÐØð c�(ÐPÐPÐPØ (ÀÐ_`Ð*aÐ*aÐbÐå”‘”€JÝ˜J¨	Ð3GÐI_Ñ`Ô`€DÝŒy‰{Œ{€HÝ	Ð
? ¨:Ñ 5Ð
?Ð
?Ð
?Ñ@Ô@Ð@å5°e¸ZÈÐUZÐ\rÑsÔsÐÝØØØØØØØØØØØðñ ô €Fð ‡M‚Mà$Ø"Ø ×(Ò(Ð)<¸bÑAÔAØ&<Ø!&ð	
ð 	
ñô ð ð €Mr   TÚuse_io_bindingc                 ó  — ddl m} |�8t          j                             |¦  «        r|                     |||¬¦  «        }n.|                     | d||¬¦  «        }|                     |¦  «         |rd |_        d |_        |S )Nr   )ÚORTPipelineForText2Image)r&   rº   T)Úexportr&   rº   )	Úoptimum.onnxruntimer¼   r/   r0   r1   r3   Úsave_pretrainedr8   r9   )r    r!   r&   r"   rº   r¼   Úpipelines          r   Úget_optimum_ort_pipelinerÁ   o  s¦   € ð =Ð<Ð<Ð<Ð<Ð<àÐ¥¤§¢°	Ñ!:Ô!:ÐØ+×;Ò;¸IÐPXÐiwÐ;ÑxÔxˆˆà+×;Ò;ØØØØ)ð	 <ñ 
ô 
ˆð 	× Ò  Ñ+Ô+Ð+àð *Ø"&ˆÔØ%)ˆÔ"à€Or   c                 ó^  ‡ ‡‡‡‡‡‡
‡‡— t          dt          ‰ ¦  «        ¦  «         ddlm} t	          ‰ |¦  «        Št          ¦   «         \  }}ˆˆˆˆˆ ˆˆˆ
ˆf	d„}t          |	||¦  «        }t          |	||¦  «        } |¦   «          t          |‰
‰‰¦  «        }g }t          |¦  «        D ]À\  }}||k    r n´t          j	        ¦   «         }‰
r ‰ d|‰‰‰‰dœ|¤Žj
        }n ‰ d|g‰z  ‰‰‰dœ|¤Žj
        }t          j	        ¦   «         }||z
  }|                     |¦  «         t          d|d›d	�¦  «         t          |¦  «        D ]#\  }}|                     |› d
|› d
|› d�¦  «         Œ$ŒÁddlm} d|‰‰‰‰‰|t          |¦  «        t!          |¦  «        z  t#          j        |¦  «        ||dœS )NzPipeline typer   )ÚORTFluxPipelinec            	      óœ   •	— ‰rd S t          ¦   «         \  } }t          |‰
‰‰¦  «        }‰
r ‰d| ‰‰‰	‰dœ|¤Ž d S  ‰d| g‰z  ‰‰‰	dœ|¤Ž d S )N©rj   rk   rl   rm   Únum_images_per_promptr£   r   r¤   )rj   ro   r¥   ry   rV   rk   rž   r;   rf   rW   r�   rl   s      €€€€€€€€€r   rp   z(run_optimum_ort_pipeline.<locals>.warmupŸ  s²   ø€ Øð 	ØˆFÝ)Ñ+Ô+Ñˆ�Ý1°(Ð<UÐW^Ð`jÑkÔkˆØ$ð 
	vØˆDð ØØØØ$)Ø&1ðð ð ðð ð ð ð ð ˆDÐu˜˜ :Ñ-°fÀEÐ_dÐuÐuÐhtÐuÐuÐuÐuÐur   rÅ   r£   rq   rr   rs   r\   rt   ru   Úoptimum_ortrw   r   )rS   ÚtypeÚ&optimum.onnxruntime.modeling_diffusionrÃ   r   r   r   r    r€   r�   r‚   rƒ   r„   r.   rv   r…   r†   r‡   rˆ   )r;   rV   re   rk   rl   rW   rz   ry   r   r‰   r�   rf   rÃ   r   r   rp   rŠ   r‹   r¥   rŒ   r�   rj   rŽ   r‚   r�   r�   r‘   r’   r“   rž   s   `` ``` `  ``                 @r   Úrun_optimum_ort_pipelinerÊ   Š  s¨  øøøøøøøøø€ õ 
ˆ/�4 ™:œ:Ñ&Ô&Ð&ØFÐFÐFÐFÐFÐFå˜˜Ñ/Ô/€Gå.Ñ0Ô0Ñ€Gˆ_ðvð vð vð vð vð vð vð vð vð vð vð vð võ& *Ð*=¸vÀ|ÑTÔTÐÝ*Ð+>ÀÈÑUÔUÐà
€F�H„H€Hå-¨oÐ?XÐZaÐcmÑnÔn€Là€LÝ˜wÑ'Ô'ð ?ð ?‰	ˆˆ6Ø�ÒÐØˆEÝœ)™+œ+ˆØ$ð 	Ø�Tð ØØØØ$)Ø&0ðð ð ðð ô ð ˆFð �Tð Ø�x *Ñ,°VÀ5Ð^cðð Øgsðð äð õ œ	™œˆØ /Ñ1ˆØ×Ò˜GÑ$Ô$Ð$ÝÐ5 Ð5Ð5Ð5Ð5Ñ6Ô6Ð6Ý! &Ñ)Ô)ð 	?ð 	?‰HˆAˆuØ�JŠJÐ/Ð=Ð=°!Ð=Ð=°aÐ=Ð=Ð=Ñ>Ô>Ð>Ð>ð	?ð 7Ð6Ð6Ð6Ð6Ð6ð  ØØØØØ Ø"Ø"Ý˜|Ñ,Ô,­s°<Ñ/@Ô/@Ñ@Ý$Ô+¨LÑ9Ô9Ø/Ø 1ðð ð r   c                 óš  — t          j         ¦   «         }t          | ||||¬¦  «        }t          j         ¦   «         }t          d||z
  › d�¦  «         |r| dz   t          |¦  «        j        z   n| }t          d||||¦  «        }t          ||||||||	|
||¬¦  «        }|                     | ||                     dd¦  «        |d	d
œ¦  «         |S )N©rº   r®   rs   r\   Úoptimumr°   r±   r_   Fr²   )	r�   rÁ   rS   r   Únamerd   rÊ   r´   rb   )r    r!   r&   rV   r"   rk   rl   rW   rz   ry   r   r‰   rº   rf   r¶   r;   r·   Úfull_model_namere   r¸   s                       r   Úrun_optimum_ortrÐ   ä  s  € õ  ”‘”€JÝ#Ø�I˜xÐ)?ÐP^ðñ ô €Dõ Œy‰{Œ{€HÝ	Ð
? ¨:Ñ 5Ð
?Ð
?Ð
?Ñ@Ô@Ð@àAJÐZ�j 3Ñ&­¨i©¬Ô)=Ñ=Ð=ÐPZ€OÝ5Ø�? J°Ð7Mñô Ðõ &ØØØØØØØØØØØðñ ô €Fð ‡M‚Mà$Ø"Ø ×(Ò(Ð)<¸bÑAÔAØ&<Ø!&ð	
ð 	
ñô ð ð €Mr   Úwork_dirrx   Úmax_batch_sizeÚnvtx_profileÚuse_cuda_graphc                 óP  ‡‡‡‡‡-— t          d¦  «         ddlm}  |¦   «          ‰|k    sJ ‚ddlm}  ||¦  «        }|                     ¦   «         }ddlm}m} ddl	m
} |j        } || ||¦  «        \  }}}}} ||d|d|||||¬	¦	  «	        Š-‰-j                             |||d
‰‰‰dddt          j                             ¦   «         ¬¦  «         ‰-                     ‰‰‰¦  «         ˆˆˆ-ˆˆfd„}t%          |
||	¦  «        }t%          |
||	¦  «        } |¦   «          t'          d|‰‰|¦  «        }g }t)          ¦   «         \  } }!t+          | ¦  «        D ]½\  }"}#|"|k    r n±t-          j        ¦   «         }$‰-                     |#g‰z  |!g‰z  ‰‰‰dd¬¦  «        \  }%}&t-          j        ¦   «         }'|'|$z
  }(|                     |(¦  «         t          d|(d›d|&› �¦  «         t+          |%¦  «        D ]#\  })}*|*                     |› d|"› d|)› d�¦  «         Œ$Œ¾‰-                     ¦   «          ddlm}+ ddlm}, i d|                     ¦   «         “dd“d|,“dd|+› d�“d|“d ‰“d!‰“d"‰“d#‰“d$|“d%|“d&t?          |¦  «        tA          |¦  «        z  “d'tC          j"        |¦  «        “d(|“d)|“d*|“d+|“S ),Nzd[I] Initializing ORT TensorRT EP accelerated StableDiffusionXL txt2img pipeline (static input shape)r   ©Úinit_trt_plugins©ÚPipelineInfo©Ú
EngineTypeÚget_engine_paths©rG   ÚDDIMF©r5   Ú
output_dirÚverboserÓ   rÒ   rÔ   Úframework_model_dirÚengine_typeé   T)Úopt_image_heightÚopt_image_widthÚopt_batch_sizeÚstatic_batchÚstatic_image_shapeÚmax_workspace_sizeÚ	device_idc                  ón   •— t          ¦   «         \  } }‰                     | g‰z  |g‰z  ‰‰‰¬¦  «         d S ©N)Údenoising_steps©r   Úrun)rj   ro   rV   rk   rÀ   rW   rl   s     €€€€€r   rp   z"run_ort_trt_static.<locals>.warmup]  sD   ø€ Ý)Ñ+Ô+Ñˆ�Ø�Š�f�X 
Ñ*¨X¨J¸Ñ,CÀVÈUÐdiˆÑjÔjÐjÐjÐjr   Úort_trtg      @r˜   ©rî   ÚguidanceÚseedúEnd2End took rr   ú seconds. Inference latency: r\   rt   ru   r    rU   r.   rx   r&   z	tensorrt(ú)r!   rk   rl   rW   rV   ry   rz   r{   r|   r}   r~   r"   r³   )#rS   Útrt_utilitiesr×   Údiffusion_modelsrÙ   Ú
short_nameÚengine_builderrÛ   rÜ   Úpipeline_stable_diffusionrG   ÚORT_TRTÚbackendÚbuild_enginesrJ   r   Úcurrent_deviceÚload_resourcesr   rd   r   r€   r�   rð   rƒ   r„   Úteardownr   rv   r.   rÎ   r…   r†   r‡   rˆ   ).rÑ   rx   rV   r"   rk   rl   rW   rz   ry   r   r‰   rÒ   rÓ   rÔ   r×   rÙ   Úpipeline_inforú   rÛ   rÜ   rG   rã   Úonnx_dirÚ
engine_dirrà   râ   r\   rp   rŠ   r‹   re   rŒ   r   r   r�   rj   rŽ   r‚   Úpipeline_timer�   r�   r‘   r’   Útrt_versionr“   rÀ   s.     ` ```                                      @r   Úrun_ort_trt_staticr    s`  øøøøø€ õ  
Ð
pÑqÔqÐqð /Ð.Ð.Ð.Ð.Ð.àÐÑÔÐà˜Ò'Ð'Ð'Ð'à-Ð-Ð-Ð-Ð-Ð-à �L Ñ)Ô)€MØ×)Ò)Ñ+Ô+€Jà;Ð;Ð;Ð;Ð;Ð;Ð;Ð;ØAÐAÐAÐAÐAÐAàÔ$€KØ?OÐ?OÐPXÐZgÐitÑ?uÔ?uÑ<€Hˆj˜*Ð&9¸1ð 'Ð&ØØØØØ!Ø%Ø%Ø/Øð
ñ 
ô 
€Hð Ô×"Ò"ØØØØ
ØØØ!ØØØÝ”*×+Ò+Ñ-Ô-ð #ñ ô ð ð  ×Ò˜F E¨:Ñ6Ô6Ð6ðkð kð kð kð kð kð kð kð kõ *Ð*=¸vÀ|ÑTÔTÐÝ*Ð+>ÀÈÑUÔUÐà
€F�H„H€Hå5°iÀÈZÐY^Ð`vÑwÔwÐà€LÝ.Ñ0Ô0Ñ€Gˆ_Ý˜wÑ'Ô'ð ?ð ?‰	ˆˆ6Ø�ÒÐØˆEÝœ)™+œ+ˆà (§¢ØˆH�zÑ!ØÐ 
Ñ*ØØØ!ØØð !-ñ !
ô !
Ñˆ�õ œ	™œˆØ /Ñ1ˆØ×Ò˜GÑ$Ô$Ð$ÝÐW˜gÐWÐWÐWÈÐWÐWÑXÔXÐXÝ! &Ñ)Ô)ð 	?ð 	?‰HˆAˆuØ�JŠJÐ/Ð=Ð=°!Ð=Ð=°aÐ=Ð=Ð=Ñ>Ô>Ð>Ð>ð	?ð ×ÒÑÔÐà3Ð3Ð3Ð3Ð3Ð3à6Ð6Ð6Ð6Ð6Ð6ðØ�m×(Ò(Ñ*Ô*ðà�-ðð 	�;ðð 	Ð. Ð.Ð.Ð.ð	ð
 	�Zðð 	�&ðð 	�ðð 	�ðð 	�jðð 	�{ðð 	�{ðð 	�3˜|Ñ,Ô,­s°<Ñ/@Ô/@Ñ@ðð 	�*Ô+¨LÑ9Ô9ðð 	Ð/ðð 	Ð 1ðð  	!Ð"8ð!ð" 	˜^ð#ð r   c                 ób  ‡‡‡‡‡‡1— t          d¦  «         ddlm} ddlm}  |¦   «          ‰|k    sJ ‚ddlm}  ||¦  «        }ddlm}m	} ddl
m} |j        } || ||¦  «        \  }}}}} ||d|d	||d
|¬¦  «        Š1‰1j                             |||d‰‰‰d
d
d	|¬¦  «         t          ‰1j                             ¦   «         ‰1j                             ¦   «         ¦  «        }|                     |¦  «        \  }}‰1j                             |¦  «         ‰1                     ‰‰‰¦  «         ˆˆˆ1ˆˆˆfd„} t)          || |
¦  «        }!t)          || |
¦  «        }" | ¦   «          t+          d|‰‰|¦  «        }#g }$t-          ¦   «         \  }%}&t/          |%¦  «        D ]¼\  }'}(|'|k    r n°t1          j        ¦   «         })‰1                     |(g‰z  |&g‰z  ‰‰‰d¬¦  «        \  }*}+t1          j        ¦   «         },|,|)z
  }-|$                     |-¦  «         t          d|-d›d|+› �¦  «         t/          |*¦  «        D ]#\  }.}/|/                     |#› d|'› d|.› d�¦  «         Œ$Œ½‰1                     ¦   «          dd l}0d|0j        d‰‰‰‰|	|t?          |$¦  «        tA          |$¦  «        z  tC          j"        |$¦  «        |!|"|dœS )Nú][I] Initializing TensorRT accelerated StableDiffusionXL txt2img pipeline (static input shape)r   ©ÚcudartrÖ   rØ   rÚ   rÝ   rÞ   FT)r5   rà   rá   rÓ   rÒ   rÔ   rã   rä   ©r  râ   r  Ú
onnx_opsetrç   rå   ræ   rè   Ústatic_shapeÚenable_all_tacticsÚtiming_cachec                  óv   •— ‰rd S t          ¦   «         \  } }‰                     | g‰z  |g‰z  ‰‰‰¬¦  «         d S rí   rï   )rj   ro   rV   rk   rÀ   rf   rW   rl   s     €€€€€€r   rp   z#run_tensorrt_static.<locals>.warmupé  sQ   ø€ Øð 	ØˆFÝ)Ñ+Ô+Ñˆ�Ø�Š�f�X 
Ñ*¨X¨J¸Ñ,CÀVÈUÐdiˆÑjÔjÐjÐjÐjr   Útrtr˜   )rî   rô   rõ   rr   rö   r\   rt   r   Údefault)rU   rx   r&   rk   rl   rW   rV   ry   rz   r{   r|   r}   r~   r³   )#rS   r   r  rø   r×   rù   rÙ   rû   rÛ   rÜ   rü   rG   ÚTRTrþ   Úload_enginesÚmaxÚmax_device_memoryÚ
cudaMallocÚactivate_enginesr  r   rd   r   r€   r�   rð   rƒ   r„   r  r   rv   r…   r†   r‡   rˆ   )2rÑ   rx   r    rV   r"   rk   rl   rW   rz   ry   r   r‰   rÒ   rÓ   rÔ   rf   r  r×   rÙ   r  rÛ   rÜ   rG   rã   r  r  rà   râ   r  r  r\   Úshared_device_memoryrp   rŠ   r‹   re   rŒ   r   r   r�   rj   rŽ   r‚   r  r�   r�   r‘   r’   r  rÀ   s2      ` ```       `                                 @r   Úrun_tensorrt_staticr  œ  sñ  øøøøøø€ õ$ 
Ð
iÑjÔjÐjàÐÐÐÐÐð /Ð.Ð.Ð.Ð.Ð.àÐÑÔÐà˜Ò'Ð'Ð'Ð'à-Ð-Ð-Ð-Ð-Ð-à �L Ñ)Ô)€Mà;Ð;Ð;Ð;Ð;Ð;Ð;Ð;ØAÐAÐAÐAÐAÐAà”.€KØJZÐJZØ�- ñKô KÑG€Hˆj˜*Ð&9¸<ð
 'Ð&ØØØØØ!Ø%ØØð	ñ 	ô 	€Hð Ô×!Ò!ØØ/ØØØ!ØØØØØ Ø!ð "ñ ô ð õ ˜HÔ,×>Ò>Ñ@Ô@À(ÔBR×BdÒBdÑBfÔBfÑgÔgÐØ$×/Ò/Ð0AÑBÔBÑ€AÐØÔ×%Ò%Ð&:Ñ;Ô;Ð;ð ×Ò˜F E¨:Ñ6Ô6Ð6ðkð kð kð kð kð kð kð kð kð kõ *Ð*=¸vÀ|ÑTÔTÐÝ*Ð+>ÀÈÑUÔUÐà
€F�H„H€Hå5°e¸ZÈÐUZÐ\rÑsÔsÐà€LÝ.Ñ0Ô0Ñ€Gˆ_Ý˜wÑ'Ô'ð ?ð ?‰	ˆˆ6Ø�ÒÐØˆEÝœ)™+œ+ˆà (§¢ØˆH�zÑ!ØÐ 
Ñ*ØØØ!Øð !-ñ !
ô !
Ñˆ�õ œ	™œˆØ /Ñ1ˆØ×Ò˜GÑ$Ô$Ð$ÝÐW˜gÐWÐWÐWÈÐWÐWÑXÔXÐXÝ! &Ñ)Ô)ð 	?ð 	?‰HˆAˆuØ�JŠJÐ/Ð=Ð=°!Ð=Ð=°aÐ=Ð=Ð=Ñ>Ô>Ð>Ð>ð	?ð ×ÒÑÔÐàÐÐÐð Ø”?ØØØØØ Ø"Ø"Ý˜|Ñ,Ô,­s°<Ñ/@Ô/@Ñ@Ý$Ô+¨LÑ9Ô9Ø/Ø 1Ø+ðð ð r   c                 óv  ‡ ‡‡‡‡‡‡‡‡‡*‡+‡,‡-‡.‡/— t          d¦  «         dd l}ddlm} ddlm} ‰Š,‰Š-‰,dz  dk    s	‰-dz  dk    rt          d‰,› d‰-› d�¦  «        ‚ |¦   «          ‰‰k    sJ ‚dd	lm} dd
l	m
Š*mŠ+ ˆ*ˆˆ+ˆˆˆˆˆˆ f	d„}ddlm}  ||¦  «        } |||¦  «        Š.t          ‰.j                             ¦   «         ‰.j                             ¦   «         ¦  «        }|                     |¦  «        \  }}‰.j                             |¦  «         ‰.                     ‰,‰-‰¦  «         dˆ,ˆ-ˆ.ˆfd„	Š/ˆˆ/ˆfd„}t)          |
||	¦  «        }t)          |
||	¦  «        } |¦   «          |                     ¦   «         }t-          d|‰‰|¦  «        }g }t/          ¦   «         \  }} t1          |¦  «        D ]¯\  }!}"|!|k    r n£t3          j        ¦   «         }# ‰/|"g‰z  | g‰z  d¬¦  «        \  }$}%t3          j        ¦   «         }&|&|#z
  }'|                     |'¦  «         t          d|'d›d|%› �¦  «         t1          |$¦  «        D ]#\  }(})|)                     |› d|!› d|(› d�¦  «         Œ$Œ°‰.                     ¦   «          |d|j        d‰‰‰‰||t=          |¦  «        t?          |¦  «        z  tA          j!        |¦  «        ||‰dœS )Nr
  r   r  rÖ   é   zCImage height and width have to be divisible by 8 but specified as: z and ú.rØ   rÚ   c                 ó®   •	— ‰	j         } ‰‰||¦  «        \  }}}}} | |d|d‰‰‰||¬¦	  «	        }|j                             |||d‰
‰‰ddd|¬¦  «         |S )NrÞ   Frß   rä   Tr  )r  rþ   r  )Úpipeline_classr  rã   r  r  rà   râ   r  rÀ   rÛ   rV   rÜ   rk   rÒ   rÓ   rÔ   rl   rÑ   s            €€€€€€€€€r   Úinit_pipelinez-run_tensorrt_static_xl.<locals>.init_pipelineK  s®   ø€ Ø ”nˆàN^ÐN^Ø�m [ñO
ô O
ÑKˆ�*˜jÐ*=¸|ð
 "�>ØØØ!ØØ%Ø)Ø)Ø 3Ø#ð

ñ 

ô 

ˆð 	Ô×%Ò%Ø!Ø 3ØØØ%Ø#Ø!ØØØ$Ø%ð 	&ñ 	
ô 	
ð 	
ð ˆr   rÝ   c           	      ó<   •— ‰                      | |‰‰‰d|¬¦  «        S ©Ng      @rò   ©rð   )rj   r   rô   Úimage_heightÚimage_widthrÀ   rW   s      €€€€r   Úrun_sd_xl_inferencez3run_tensorrt_static_xl.<locals>.run_sd_xl_inference{  s3   ø€ Ø�|Š|ØØØØØ!ØØð ñ 
ô 
ð 	
r   c                  óZ   •— ‰rd S t          ¦   «         \  } } ‰| g‰z  |g‰z  ¦  «         d S ©Nrn   ©rj   ro   rV   r(  rf   s     €€€r   rp   z&run_tensorrt_static_xl.<locals>.warmup†  óG   ø€ Øð 	ØˆFÝ)Ñ+Ô+Ñˆ�ØÐ˜V˜H zÑ1°H°:À
Ñ3JÑKÔKÐKÐKÐKr   r  r˜   ©rô   rõ   rr   rö   r\   ú.pngr   r  ©r    rU   rx   r&   rk   rl   rW   rV   ry   rz   r{   r|   r}   r~   r³   r*  )"rS   r   r   r  rø   r×   Ú
ValueErrorrù   rÙ   rû   rÛ   rÜ   rü   rG   r  rþ   r  r  r  r  r   rÎ   rd   r   r€   r�   rƒ   r„   r  rv   r…   r†   r‡   rˆ   )0rÑ   rx   rV   r"   rk   rl   rW   rz   ry   r   r‰   rÒ   rÓ   rÔ   rf   r  r  r×   rÙ   r"  rG   r  r  r\   r  rp   rŠ   r‹   r    re   rŒ   r   r   r�   rj   rŽ   r‚   r  r�   r�   r‘   r’   rÛ   rÜ   r&  r'  rÀ   r(  s0   ` ` ```    ````                           @@@@@@r   Úrun_tensorrt_static_xlr1  $  s  øøøøøøøøøøøøøøø€ õ" 
Ð
iÑjÔjÐjàÐÐÐØÐÐÐÐÐØ.Ð.Ð.Ð.Ð.Ð.ð €LØ€KØ�aÑ˜1ÒÐ ¨a¡°1Ò 4Ð 4ÝØsÐR^ÐsÐsÐepÐsÐsÐsñ
ô 
ð 	
ð
 ÐÑÔÐà˜Ò'Ð'Ð'Ð'à-Ð-Ð-Ð-Ð-Ð-Ø;Ð;Ð;Ð;Ð;Ð;Ð;Ð;ð!ð !ð !ð !ð !ð !ð !ð !ð !ð !ð !ð !ð !ðF BÐAÐAÐAÐAÐAà �L Ñ)Ô)€MØˆ}Ð4°mÑDÔD€Hå˜HÔ,×>Ò>Ñ@Ô@À(ÔBR×BdÒBdÑBfÔBfÑgÔgÐØ$×/Ò/Ð0AÑBÔBÑ€AÐØÔ×%Ò%Ð&:Ñ;Ô;Ð;ð ×Ò˜L¨+°zÑBÔBÐBð	
ð 	
ð 	
ð 	
ð 	
ð 	
ð 	
ð 	
ð 	
ðLð Lð Lð Lð Lð Lð Lõ *Ð*=¸vÀ|ÑTÔTÐÝ*Ð+>ÀÈÑUÔUÐà
€F�H„H€Hà×#Ò#Ñ%Ô%€JÝ5°e¸ZÈÐUZÐ\rÑsÔsÐà€LÝ.Ñ0Ô0Ñ€Gˆ_Ý˜wÑ'Ô'ð ?ð ?‰	ˆˆ6Ø�ÒÐØˆEÝœ)™+œ+ˆà 3Ð 3°V°H¸zÑ4IÈOÐK\Ð_iÑKiÐpsÐ tÑ tÔ tÑˆ�Ýœ	™œˆØ /Ñ1ˆØ×Ò˜GÑ$Ô$Ð$ÝÐW˜gÐWÐWÐWÈÐWÐWÑXÔXÐXÝ! &Ñ)Ô)ð 	?ð 	?‰HˆAˆuØ�JŠJÐ/Ð=Ð=°!Ð=Ð=°aÐ=Ð=Ð=Ñ>Ô>Ð>Ð>ð	?ð ×ÒÑÔÐð !ØØ”?ØØØØØ Ø"Ø"Ý˜|Ñ,Ô,­s°<Ñ/@Ô/@Ñ@Ý$Ô+¨LÑ9Ô9Ø/Ø 1Ø+ðð ð r   c                 óø  ‡‡‡‡‡‡%‡&— ddl m} ddlm}  |||j        | ‰‰||‰¬¦  «        Š%‰|k    sJ ‚‰%                     ‰‰‰¦  «         dˆˆ%ˆˆfd„	Š&ˆˆ&ˆfd„}t          |
||	¦  «        }t          |
||	¦  «        } |¦   «          ‰%j                             ¦   «         }t          d|‰‰|¦  «        }g }t          ¦   «         \  }}t          |¦  «        D ]Á\  }}||k    r nµt          j        ¦   «         } ‰&|g‰z  |g‰z  d¬	¦  «        \  }}t          j        ¦   «         }||z
  }|                     |¦  «         t          d
|d›d|› �¦  «         t          |¦  «        D ]5\  } }!|› d|› d| › d�}"|!                     |"¦  «         t          d|"¦  «         Œ6ŒÂ‰%                     ¦   «          ddlm}# ddlm}$ |d|$d|#› d�‰‰‰‰||t)          |¦  «        t+          |¦  «        z  t-          j        |¦  «        |||dœS )Nr   )Úinitialize_pipeline)rÛ   )rx   rã   rÑ   rk   rl   rÔ   rÒ   rç   c           	      ó<   •— ‰                      | |‰‰‰d|¬¦  «        S r$  r%  )rj   r   rô   rk   rÀ   rW   rl   s      €€€€r   r(  z+run_ort_trt_xl.<locals>.run_sd_xl_inferenceÝ  s3   ø€ Ø�|Š|ØØØØØ!ØØð ñ 
ô 
ð 	
r   c                  óZ   •— ‰rd S t          ¦   «         \  } } ‰| g‰z  |g‰z  ¦  «         d S r*  rn   r+  s     €€€r   rp   zrun_ort_trt_xl.<locals>.warmupè  r,  r   rñ   r˜   r-  rõ   rr   rö   r\   r.  zImage saved toru   r.   r   r÷   r/  r*  )Ú
demo_utilsr3  rû   rÛ   rý   r  r   r  rÎ   rd   r   r€   r�   rƒ   rS   r„   r  r   rv   r.   r…   r†   r‡   rˆ   )'rÑ   rx   rV   r"   rk   rl   rW   rz   ry   r   r‰   rÒ   rÓ   rÔ   rf   r3  rÛ   rp   rŠ   r‹   r    re   rŒ   r   r   r�   rj   rŽ   r‚   r  r�   r�   r‘   r’   Úfilenamer  r“   rÀ   r(  s'     ` ```       `                      @@r   Úrun_ort_trt_xlr8  º  s
  øøøøøøø€ ð" /Ð.Ð.Ð.Ð.Ð.Ø)Ð)Ð)Ð)Ð)Ð)à"Ð"ØØÔ&ØØØØ%Ø%Ø!ð	ñ 	ô 	€Hð ˜Ò'Ð'Ð'Ð'à×Ò˜F E¨:Ñ6Ô6Ð6ð	
ð 	
ð 	
ð 	
ð 	
ð 	
ð 	
ð 	
ð 	
ðLð Lð Lð Lð Lð Lð Lõ *Ð*=¸vÀ|ÑTÔTÐÝ*Ð+>ÀÈÑUÔUÐà
€F�H„H€HàÔ'×,Ò,Ñ.Ô.€JÝ5°iÀÈZÐY^Ð`vÑwÔwÐà€LÝ.Ñ0Ô0Ñ€Gˆ_Ý˜wÑ'Ô'ð .ð .‰	ˆˆ6Ø�ÒÐØˆEÝœ)™+œ+ˆà 3Ð 3°V°H¸zÑ4IÈOÐK\Ð_iÑKiÐpsÐ tÑ tÔ tÑˆ�Ýœ	™œˆØ /Ñ1ˆØ×Ò˜GÑ$Ô$Ð$ÝÐW˜gÐWÐWÐWÈÐWÐWÑXÔXÐXÝ! &Ñ)Ô)ð 	.ð 	.‰HˆAˆuØ/Ð=Ð=°!Ð=Ð=°aÐ=Ð=Ð=ˆHØ�JŠJ�xÑ Ô Ð ÝÐ" HÑ-Ô-Ð-Ð-ð	.ð
 ×ÒÑÔÐà3Ð3Ð3Ð3Ð3Ð3à6Ð6Ð6Ð6Ð6Ð6ð !ØØØ-˜{Ð-Ð-Ð-ØØØØ Ø"Ø"Ý˜|Ñ,Ô,­s°<Ñ/@Ô/@Ñ@Ý$Ô+¨LÑ9Ô9Ø/Ø 1Ø+ðð ð r   c                 óR  — dt           j        j        _        dt           j        j        _        t          j        d¦  «         t          j        ¦   «         }t          | |||¦  «        }t          j        ¦   «         }t          d||z
  › d�¦  «         t          d| |||¦  «        }|sFt          j
        ¦   «         5  t          ||||||||	|
||¬¦  «        }d d d ¦  «         n# 1 swxY w Y   nt          ||||||||	|
||¬¦  «        }|                     | d |rdn|rdnd	|dd
œ¦  «         |S )NTFr®   rs   rJ   r°   rN   Úxformersr  r²   )rJ   ÚbackendsÚcudnnÚenabledÚ	benchmarkr¦   r�   rT   rS   rd   Úinference_moder¨   r´   )r    rV   r"   r=   r>   rk   rl   rW   rz   ry   r   r‰   rf   r¶   r;   r·   re   r¸   s                     r   Ú	run_torchr@  "  s¶  € ð $(…E„NÔÔ Ø%)…E„NÔÔ"å	Ô˜5Ñ!Ô!Ð!å”‘”€JÝ˜jÐ*@ÐBVÐXdÑeÔe€DÝŒy‰{Œ{€HÝ	Ð
? ¨:Ñ 5Ð
?Ð
?Ð
?Ñ@Ô@Ð@å5°g¸zÈ:ÐW\Ð^tÑuÔuÐàð 
ÝÔ!Ñ#Ô#ð 	ð 	Ý'ØØØ%ØØØØØØØ#Ø'ðñ ô ˆFð	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	øõ $ØØØ!ØØØØØØØØ#ð
ñ 
ô 
ˆð ‡M‚Mà$ØØ%9Ðh˜	˜	È\Ð?h¸z¸zÐ_hØ&<Ø!&ð	
ð 	
ñô ð ð €Ms   Â7CÃC"Ã%C"c                  ó’  — t          j        ¦   «         } |                      dddt          dg d¢d¬¦  «         |                      dd	dt          d
t	          t
                               ¦   «         ¦  «        d¬¦  «         |                      dddd¬¦  «         |                      dddt          t	          t                               ¦   «         ¦  «        dd¬¦  «         |                      dddt          d d¬¦  «         |                      dddt          dd¬¦  «         |                      dddd¬ ¦  «         |                      d¬!¦  «         |                      d"ddd#¬ ¦  «         |                      d¬$¦  «         |                      d%ddd&¬ ¦  «         |                      d¬'¦  «         |                      d(ddd)¬ ¦  «         |                      d¬*¦  «         |                      d+ddd,¬ ¦  «         |                      d¬-¦  «         |                      d.d/t          d0g d1¢d2¬3¦  «         |                      d4dt          d5d6¬¦  «         |                      d7dt          d5d8¬¦  «         |                      d9d:dt          d;d<¬¦  «         |                      d=d>dt          d?d@¬¦  «         |                      dAdBdt          t          d0dC¦  «        dDdE¬¦  «         |                      dFdGdt          t          d0dH¦  «        dIdJ¬¦  «         |                      dKdLdddM¬ ¦  «         |                      d¬N¦  «         |                      ¦   «         }|S )ONz-ez--engineFr.   )r.   rÍ   rJ   r   z-Engines to benchmark. Default is onnxruntime.)ÚrequiredrÈ   r  ÚchoicesÚhelpz-rz
--providerr   z8Provider to benchmark. Default is CUDAExecutionProvider.z-tz--tuningÚ
store_truezsEnable TunableOp and tuning. This will incur longer warmup latency, and is mandatory for some operators of ROCm EP.)ÚactionrD  z-vz	--versionr   z>Stable diffusion version like 1.5, 2.0 or 2.1. Default is 1.5.)rB  rÈ   rC  r  rD  z-pz
--pipelinez[Directory of saved onnx pipeline. It could be the output directory of optimize_pipeline.py.)rB  rÈ   r  rD  z-wz
--work_dirr  z?Root directory to save exported onnx models, built engines etc.z--enable_safety_checkerzEnable safety checker)rB  rF  rD  )Úenable_safety_checkerz--enable_torch_compilez#Enable compile unet for PyTorch 2.0)r=   z--use_xformerszUse xformers for PyTorch)r>   z--use_io_bindingzUse I/O Binding for Optimum.rÌ   z--skip_warmupz
No warmup.r°   z-bz--batch_sizer«   )r«   é   é   é   r  é
   é   é    z)Number of images per batch. Default is 1.)rÈ   r  rC  rD  z--heighti   z$Output image height. Default is 512.z--widthz#Output image width. Default is 512.z-sz--stepsé2   zNumber of steps. Default is 50.z-nz--num_promptsrK  z!Number of prompts. Default is 10.z-cz--batch_counté   é   z(Number of batches to test. Default is 5.z-mz--max_trt_batch_sizerL  rJ  zdMaximum batch size for TensorRT. Change the value may trigger TensorRT engine rebuild. Default is 4.z-gz--enable_cuda_graphz/Enable Cuda Graph. Requires onnxruntime >= 1.16)r³   )ÚargparseÚArgumentParserÚadd_argumentÚstrÚlistÚ	PROVIDERSÚkeysÚ	SD_MODELSÚset_defaultsÚintÚrangeÚ
parse_args)ÚparserÚargss     r   Úparse_argumentsr_  g  sx  € ÝÔ$Ñ&Ô&€Fà
×ÒØØØÝØØ?Ð?Ð?Ø<ð ñ ô ð ð ×ÒØØØÝØÝ•Y—^’^Ñ%Ô%Ñ&Ô&ØGð ñ ô ð ð ×ÒØØØðað	 ñ ô ð ð ×ÒØØØÝÝ•Y—^’^Ñ%Ô%Ñ&Ô&ØØMð ñ ô ð ð ×ÒØØØÝØØjð ñ ô ð ð ×ÒØØØÝØØNð ñ ô ð ð ×ÒØ!ØØØ$ð	 ñ ô ð ð ×Ò¨eÐÑ4Ô4Ð4à
×ÒØ ØØØ2ð	 ñ ô ð ð ×Ò¨UÐÑ3Ô3Ð3à
×ÒØØØØ'ð	 ñ ô ð ð ×Ò UÐÑ+Ô+Ð+à
×ÒØØØØ+ð	 ñ ô ð ð ×Ò uÐÑ-Ô-Ð-à
×ÒØØØØð	 ñ ô ð ð ×Ò EÐÑ*Ô*Ð*à
×ÒØØÝØØ+Ð+Ð+Ø8ð ñ ô ð ð ×ÒØØÝØØ3ð ñ ô ð ð ×ÒØØÝØØ2ð ñ ô ð ð ×ÒØØØÝØØ.ð ñ ô ð ð ×ÒØØØÝØØ0ð ñ ô ð ð ×ÒØØØÝÝ�a˜‘”ØØ7ð ñ ô ð ð ×ÒØØØÝÝ�a˜‘”ØØsð ñ ô ð ð ×ÒØØØØØ>ð ñ ô ð ð ×Ò¨%ÐÑ0Ô0Ð0à×ÒÑÔ€Dà€Kr   c                 óì   ‡— dd l }|                     t          j        ¦   «         ¦  «        }|                     ¦   «         D ]3Š| rt          ˆfd„dD ¦   «         ¦  «        rt          ‰j        ¦  «         Œ4d S )Nr   c              3   ó*   •K  — | ]}|‰j         v V — Œd S r*  )r0   )Ú.0ÚxÚlibs     €r   ú	<genexpr>z)print_loaded_libraries.<locals>.<genexpr>  s)   øè è € Ð)`Ð)`¸A¨!¨s¬x¨-Ð)`Ð)`Ð)`Ð)`Ð)`Ð)`r   )ÚlibcuÚlibnvr   )ÚpsutilÚProcessr/   ÚgetpidÚmemory_mapsÚanyrS   r0   )Úcuda_related_onlyrh  Úprd  s      @r   Úprint_loaded_librariesro    s   ø€ Ø€M€M€Mà�Š•r”y‘{”{Ñ#Ô#€AØ�}Š}‰Œð ð ˆØ!ð 	¥cÐ)`Ð)`Ð)`Ð)`ÐA_Ð)`Ñ)`Ô)`Ñ&`Ô&`ð 	Ý�#”(‰OŒOˆOøðð r   c                  ó  — t          ¦   «         } t          | ¦  «         | j        dk    rË| j        dv rdt          j        d<   ddlm} ddlm} | 	                    |¦  «        | 	                    d¦  «        k    rdt          j        d	<   | j
        re| j        dk    r| j        d
v r| j        �t          d¦  «        ‚| 	                    |¦  «        | 	                    d¦  «        k     rt          d¦  «        ‚t          j        d¬¦  «         | j        dk    rdnd}t!          |d ¦  «        }t          d|¦  «         t"          | j                 }t$          | j                 }| j        dk    ræ| j        dk    rÛd| j        v rft          d¦  «         t'          | j        | j        | j        d| j        | j        | j        | j        | j        ||| j        d| j
        | j        ¬¦  «        }�nt          d¦  «         t;          | j        | j        | j        | j         | j        | j        | j        | j        | j        ||| j        d| j
        | j        ¬¦  «        }�n›| j        dk    rp|dk    rjd| j        v rdt          j        d	<   t?          || j        || j        | j         | j        | j        | j        | j        | j        ||| j         | j        ¬¦  «        }�n | j        dk    r¡| j        r$t          j!         "                    | j        ¦  «        s
J d¦   «         ‚t          d|› d| j#        › �¦  «         tI          || j        || j        | j         | j        | j        | j        | j        | j        ||| j#        | j        ¬ ¦  «        }�nt| j        dk    rnd| j        v ret          d!¦  «         tK          | j        | j        | j        d| j        | j        | j        | j        | j        ||| j        d| j
        | j        ¬¦  «        }nû| j        dk    r�t          d"¦  «         tM          d>i d#| j        “d$| j        “d%|“d&| j        “d'd“d(| j        “d)| j        “d*| j        “d+| j        “d,| j        “d-|“d.|“d/| j        “d0d“d1| j
        “d2| j        “Ž}not          d3| j'        › d4| j(        › d5�¦  «         tS          || j        | j         | j'        | j(        | j        | j        | j        | j        | j        ||| j        ¬6¦  «        }t          |¦  «         tU          d7d8d9¬:¦  «        5 }g d;¢}	tW          j,        ||	¬<¦  «        }
|
 -                    ¦   «          |
 .                    |¦  «         d d d ¦  «         n# 1 swxY w Y   | j        d=k    rt_          | j        d
v ¦  «         d S d S )?Nr.   )r   Ú1ÚORT_DISABLE_TRT_FLASH_ATTENTIONr   )rx   ru   z1.16.0Ú!ORT_ENABLE_FUSED_CAUSAL_ATTENTION)r   r   z:The stable diffusion pipeline does not support CUDA graph.z1.16z.CUDA graph requires ONNX Runtime 1.16 or laterz%(funcName)20s: %(message)s)Úfmtr   r   z&GPU memory used before loading models:r   ÚxlzNTesting Txt2ImgXLPipeline with static input shape. Backend is ORT TensorRT EP.TF)rÑ   rx   rV   r"   rk   rl   rW   rz   ry   r   r‰   rÒ   rÓ   rÔ   rf   zLTesting Txt2ImgPipeline with static input shape. Backend is ORT TensorRT EP.rÍ   r   )r    r!   r&   rV   r"   rk   rl   rW   rz   ry   r   r‰   rº   rf   z?--pipeline should be specified for the directory of ONNX modelsz/Testing diffusers StableDiffusionPipeline with z provider and tuning=)r    r!   r&   rV   r"   rk   rl   rW   rz   ry   r   r‰   r©   rf   zGTesting Txt2ImgXLPipeline with static input shape. Backend is TensorRT.zETesting Txt2ImgPipeline with static input shape. Backend is TensorRT.rÑ   rx   r    rV   r"   rk   rl   rW   rz   ry   r   r‰   rÒ   rÓ   rÔ   rf   zNTesting Txt2ImgPipeline with dynamic input shape. Backend is PyTorch: compile=z, xformers=r  )r    rV   r"   r=   r>   rk   rl   rW   rz   ry   r   r‰   rf   zbenchmark_result.csvÚar_   )rD   Únewline)r    r!   rU   rx   r&   r"   rk   rl   rW   rV   ry   rz   r{   r|   r}   r~   r³   )Ú
fieldnamesr«   r   )0r_  rS   rU   rx   r/   ÚenvironÚ	packagingr.   rv   Úparser³   r&   rÀ   r0  ÚcoloredlogsÚinstallr   rX  rV  r8  rÑ   rV   rk   rl   rW   rz   ry   Úmax_trt_batch_sizerf   r  rG  rÐ   rº   r0   Úisdirr©   r¹   r1  r  r=   r>   r@  ÚopenÚcsvÚ
DictWriterÚwriteheaderÚwriterowro  )r^  rx   r“   r‰   r   Úsd_modelr&   r¸   Úcsv_fileÚcolumn_namesÚ
csv_writers              r   Úmainr‰     sû  € ÝÑÔ€DÝ	ˆ$�K„K€Kà„{�mÒ#Ð#ØŒ<˜7Ð"Ð"ð =@�BŒJÐ8Ñ9à%Ð%Ð%Ð%Ð%Ð%à:Ð:Ð:Ð:Ð:Ð:à�=Š=˜Ñ%Ô%¨¯ª°xÑ)@Ô)@Ò@Ð@ð ?B�BŒJÐ:Ñ;àÔ!ð 	SØ”K =Ò0Ð0°T´]ÐFZÐ5ZÐ5ZÐ_cÔ_lÐ_tÝ Ð!]Ñ^Ô^Ð^à�}Š}˜[Ñ)Ô)¨G¯MªM¸&Ñ,AÔ,AÒAÐAÝ Ð!QÑRÔRÐRåÔÐ9Ð:Ñ:Ô:Ð:à$(¤M°VÒ$;Ð$;˜&˜&ÀÐå%Ð&9¸4Ñ@Ô@€LÝ	Ð
2°LÑAÔAÐAå˜œÔ&€HÝ˜œÔ'€HØ„{�mÒ#Ð#¨¬¸Ò(CÐ(CØ�4”<ÐÐÝÐbÑcÔcÐcÝ#ØœØœØœ?Ø'+Ø”{Ø”jØ”jØ Ô,Ø Ô,Ø)Ø$7Ø#Ô6Ø"Ø#Ô5Ø Ô,ðñ ô ˆF‰Fõ$ Ð`ÑaÔaÐaÝ'ØœØœØœ?Ø+/Ô+EÐ'EØ”{Ø”jØ”jØ Ô,Ø Ô,Ø)Ø$7Ø#Ô6Ø"Ø#Ô5Ø Ô,ðñ ô ˆF‰Fð" 
Œ˜	Ò	!Ð	! hÐ2IÒ&IÐ&IØ�4”<ÐÐØ>A�BŒJÐ:Ñ;å ØØ”mØØ”Ø'+Ô'AÐ#AØ”;Ø”*Ø”*ØÔ(ØÔ(Ø%Ø 3ØÔ.ØÔ(ð
ñ 
ô 
ˆ‰ð  
Œ˜Ò	%Ð	%ØŒ}ð 	
¥¤§¢¨t¬}Ñ!=Ô!=ð 	
ð 	
ØMñ	
ô 	
Ð=õ 	ÐlÀÐlÐlÐ_cÔ_jÐlÐlÑmÔmÐmÝØØ”mØØ”Ø'+Ô'AÐ#AØ”;Ø”*Ø”*ØÔ(ØÔ(Ø%Ø 3Ø”;ØÔ(ð
ñ 
ô 
ˆ‰ð  
Œ˜
Ò	"Ð	" t¨t¬|Ð';Ð';ÝÐWÑXÔXÐXÝ'Ø”]Ø”LØ”Ø#'Ø”;Ø”*Ø”*ØÔ(ØÔ(Ø%Ø 3ØÔ2ØØÔ1ØÔ(ð
ñ 
ô 
ˆˆð" 
Œ˜
Ò	"Ð	"ÝÐUÑVÔVÐVÝ$ð 
ð 
ð 
Ø”]�]ð
à”L�Lð
ð  �xð
ð ”�ð	
ð
 $( 4ð
ð ”;�;ð
ð ”*�*ð
ð ”*�*ð
ð Ô(Ð(ð
ð Ô(Ð(ð
ð &˜ð
ð !4Ð 3ð
ð  Ô2Ð2ð
ð ˜ð
ð  Ô1Ð1ð
ð  Ô(Ð(ð!
ˆˆõ& 	ð XÐ]aÔ]vð  Xð  Xð  DHô  DUð  Xð  Xð  Xñ	
ô 	
ð 	
õ ØØ”Ø'+Ô'AÐ#AØ!%Ô!:ØÔ*Ø”;Ø”*Ø”*ØÔ(ØÔ(Ø%Ø 3ØÔ(ð
ñ 
ô 
ˆõ  
ˆ&�M„M€Må	Ð$¨3¸Ð	;Ñ	;Ô	;ð $¸xð
ð 
ð 
ˆõ& ”^ H¸ÐFÑFÔFˆ
Ø×ÒÑ Ô Ð Ø×Ò˜FÑ#Ô#Ð#ð-$ð $ð $ñ $ô $ð $ð $ð $ð $ð $ð $øøøð $ð $ð $ð $ð2 „z�Q‚€Ý˜tœ}Ð0DÐDÑEÔEÐEÐEÐEð €s   Ô	AUÕUÕ UÚ__main__r*  )F)r   TF)FF)FT)FTF)T)-rQ  r�  r/   r‡   Úsysr�   Úpathlibr   Ú__init__r|  rJ   Úbenchmark_helperr   rX  rV  r   r   r   rT  Úboolr<   rT   rZ  rd   r”   Údictr    r¨   r¹   rÁ   rÊ   rÐ   r  r  r1  r8  r@  r_  ro  r‰  Ú__name__Ú	tracebackÚ	ExceptionÚprint_exceptionÚexc_infor   r   r   ú<module>r–     sâ  ðð €€€Ø 
€
€
€
Ø 	€	€	€	Ø Ð Ð Ð Ø 
€
€
€
Ø €€€Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð ð €€€Ø +Ð +Ð +Ð +Ð +Ð +ð ,Ø+Ø-Ø;Ø=Ø5Ø4Ø1Ø-ð
ð 
€	ð $Ø#Ø+Ø+ð	ð €	ð$ð $ð $ð&ð ð ðhð hð hð hð ð °ð ÐX\ð ð ð ð ð<* 3ð *Àð *Ð\`ð *Ðptð *ð *ð *ð *ðZo cð o°sð oÈð oÐTWð oÐquð oð oð oð oð  ðHð HàðHð ðHð ðHð Hð Hð HðVÐcgð ð ð ð ð< ðFð FàðFð ðFð Fð Fð Fðn ð1ð 1Øð1àð1ð ð1ð ð	1ð
 !ð1ð ð1ð ð1ð ð1ð ð1ð ð1ð ð1ð ð1ð 1ð 1ð 1ðn %Ø#'Ø ðð Øðàðð !ð	ð
 ðð ð ð ðL $ØðWð WàðWð ðWð Wð Wð WðN !Øð2ð 2Øð2àð2ð ð2ð ð	2ð
 !ð2ð ð2ð ð2ð ð2ð ð2ð ð2ð ð2ð ð2ð 2ð 2ð 2ðD Øð@ð @Øð@àð@ð ð@ð !ð	@ð
 ð@ð ð@ð ð@ð ð@ð ð@ð ð@ð ð@ð ð@ð @ð @ð @ðb ØØð!Eð EØðEàðEð ðEð ð	Eð
 !ðEð ðEð ðEð ðEð ðEð ðEð ðEð ðEð ðEð  ð!Eð Eð Eð Eðj ØØðSð SØðSàðSð ðSð !ð	Sð
 ðSð ðSð ðSð ðSð ðSð ðSð ðSð ðSð Sð Sð SðF ØØðeð eØðeàðeð ðeð !ð	eð
 ðeð ðeð ðeð ðeð ðeð ðeð ðeð ðeð eð eð eðj ðBð BØðBàðBð !ðBð ð	Bð
 ðBð ðBð ðBð ðBð ðBð ðBð ðBð Bð Bð BðJmð mð mð`ð ð ð ðJFð JFð JFðZ ˆzÒÐØÐÐÐð3Øˆ‰ŒˆˆˆøØð 3ð 3ð 3Ø!ˆ	Ô! < 3¤<¡>¤>Ð2Ð2Ð2Ð2Ð2ð3øøøð Ðs   Æ
F& Æ&GÇG