§
    l¬¸jq$  ã                  ó,  — d dl mZ d dlZd dlmZ d dlmZ  G d„ d¦  «        Zedk    rhdZ	dZ
d	Z e¦   «         Ze                     ¦   «          e                     ee	e
¬
¦  «        xZr, ede› �¦  «         d„ eD ¦   «         Z ede› �¦  «         dS dS dS )é    )ÚannotationsN)ÚAny)ÚLOGGERc                  óJ   — e Zd ZdZd„ Zd„ Zd„ Zd„ Zdd
„Zd„ Z		 	 	 	 ddd„Z
dS )ÚGPUInfoaT  Manages NVIDIA GPU information via pynvml with robust error handling.

    Provides methods to query detailed GPU statistics (utilization, memory, temp, power) and select the most idle GPUs
    based on configurable criteria. It safely handles the absence or initialization failure of the pynvml library by
    logging warnings and disabling related features, preventing application crashes.

    Includes fallback logic using `torch.cuda` for basic device counting if NVML is unavailable during GPU
    selection. Manages NVML initialization and shutdown internally.

    Attributes:
        pynvml (module | None): The `pynvml` module if successfully imported and initialized, otherwise `None`.
        nvml_available (bool): Indicates if `pynvml` is ready for use. True if `nvmlInit()` succeeded, False otherwise.
        gpu_stats (list[dict[str, Any]]): A list of dictionaries, each holding stats for one GPU, populated on
        initialization and by `refresh_stats()`. Keys include: 'index', 'name', 'utilization' (%), 'memory_used' (MiB),
            'memory_total' (MiB), 'memory_free' (MiB), 'temperature' (C), 'power_draw' (W), 'power_limit' (W or 'N/A').
            Empty if NVML is unavailable or queries fail.

    Methods:
        refresh_stats: Refresh the internal gpu_stats list by querying NVML.
        print_status: Print GPU status in a compact table format using current stats.
        select_idle_gpu: Select the most idle GPUs based on utilization and free memory.
        shutdown: Shut down NVML if it was initialized.

    Examples:
        Initialize GPUInfo and print status
        >>> gpu_info = GPUInfo()
        >>> gpu_info.print_status()

        Select idle GPUs with minimum memory requirements
        >>> selected = gpu_info.select_idle_gpu(count=2, min_memory_fraction=0.2)
        >>> print(f"Selected GPU indices: {selected}")
    c                ó  — d| _         d| _        g | _        	 ddl }|| _         |                     ¦   «          d| _        |                      ¦   «          dS # t
          $ r"}t          j        d|› �¦  «         Y d}~dS d}~ww xY w)z?Initialize GPUInfo, attempting to import and initialize pynvml.NFr   Tz1Failed to initialize pynvml, GPU stats disabled: )ÚpynvmlÚnvml_availableÚ	gpu_statsÚnvmlInitÚrefresh_statsÚ	Exceptionr   Úwarning)Úselfr	   Úes      úl/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/ultralytics/utils/autodevice.pyÚ__init__zGPUInfo.__init__-   s¬   € à"&ˆŒØ$)ˆÔØ/1ˆŒð	TØˆMˆMˆMà ˆDŒKØ�OŠOÑÔÐØ"&ˆDÔØ×ÒÑ Ô Ð Ð Ð øÝð 	Tð 	Tð 	TÝŒNÐRÈqÐRÐRÑSÔSÐSÐSÐSÐSÐSÐSÐSøøøøð	Tøøøs   —:A Á
A?ÁA:Á:A?c                ó.   — |                       ¦   «          dS )z>Ensure NVML is shut down when the object is garbage collected.N)Úshutdown©r   s    r   Ú__del__zGPUInfo.__del__=   s   € à�Š‰Œˆˆˆó    c                óŽ   — | j         r;| j        r6	 | j                             ¦   «          n# t          $ r Y nw xY wd| _         dS dS dS )z%Shut down NVML if it was initialized.FN)r
   r	   ÚnvmlShutdownr   r   s    r   r   zGPUInfo.shutdownA   st   € àÔð 	( 4¤;ð 	(ðØ”×(Ò(Ñ*Ô*Ð*Ð*øÝð ð ð Ø�ðøøøà"'ˆDÔÐÐð	(ð 	(ð 	(ð 	(s   �* ª
7¶7c                ó<  ‡ — g ‰ _         ‰ j        r‰ j        sdS 	 ‰ j                             ¦   «         }‰ j                              ˆ fd„t          |¦  «        D ¦   «         ¦  «         dS # t          $ r)}t          j        d|› �¦  «         g ‰ _         Y d}~dS d}~ww xY w)z5Refresh the internal gpu_stats list by querying NVML.Nc              3  óB   •K  — | ]}‰                      |¦  «        V — Œd S )N)Ú_get_device_stats)Ú.0Úir   s     €r   ú	<genexpr>z(GPUInfo.refresh_stats.<locals>.<genexpr>R   s1   øè è € Ð!YÐ!YÀ $×"8Ò"8¸Ñ";Ô";Ð!YÐ!YÐ!YÐ!YÐ!YÐ!Yr   zError during device query: )	r   r
   r	   ÚnvmlDeviceGetCountÚextendÚranger   r   r   )r   Údevice_countr   s   `  r   r   zGPUInfo.refresh_statsJ   s¿   ø€ àˆŒØÔ"ð 	¨$¬+ð 	ØˆFð	 Øœ;×9Ò9Ñ;Ô;ˆLØŒN×!Ò!Ð!YÐ!YÐ!YÐ!YÅUÈ<ÑEXÔEXÐ!YÑ!YÔ!YÑYÔYÐYÐYÐYøÝð 	 ð 	 ð 	 ÝŒNÐ<¸Ð<Ð<Ñ=Ô=Ð=ØˆDŒNˆNˆNˆNˆNˆNˆNøøøøð	 øøøs   šAA( Á(
BÁ2BÂBÚindexÚintÚreturnúdict[str, Any]c                óü  — | j                              |¦  «        }| j                              |¦  «        }| j                              |¦  «        }dddœd„}t	          | j         dd¦  «        }|| j                              |¦  «        |r|j        nd|r
|j        dz	  nd|r
|j        dz	  nd|r
|j	        dz	  nd || j         j
        ||¦  «         || j         j        |d¬¦  «         || j         j        |d¬¦  «        d	œ	S )
z"Get stats for a single GPU device.éÿÿÿÿé   )ÚdefaultÚdivisorc               ó„   — 	  | |Ž }|dk    r!t          |t          t          f¦  «        r||z  n|S # t          $ r |cY S w xY w)Nr+   )Ú
isinstancer&   Úfloatr   )Úfuncr,   r-   ÚargsÚvals        r   Úsafe_getz+GPUInfo._get_device_stats.<locals>.safe_get]   s`   € ðØ�d˜D�k�Ø)0°Aª¨½*ÀSÍ3ÕPUÈ,Ñ:WÔ:W¨�s˜g‘~�~Ð]`Ð`øÝð ð ð Ø���ðøøøs   ‚-0 °?¾?ÚNVML_TEMPERATURE_GPUé   iè  )r-   )	r%   ÚnameÚutilizationÚmemory_usedÚmemory_totalÚmemory_freeÚtemperatureÚ
power_drawÚpower_limit)r	   ÚnvmlDeviceGetHandleByIndexÚnvmlDeviceGetMemoryInfoÚnvmlDeviceGetUtilizationRatesÚgetattrÚnvmlDeviceGetNameÚgpuÚusedÚtotalÚfreeÚnvmlDeviceGetTemperatureÚnvmlDeviceGetPowerUsageÚnvmlDeviceGetEnforcedPowerLimit)r   r%   ÚhandleÚmemoryÚutilr4   Ú	temp_types          r   r   zGPUInfo._get_device_statsW   s2  € à”×7Ò7¸Ñ>Ô>ˆØ”×4Ò4°VÑ<Ô<ˆØŒ{×8Ò8¸Ñ@Ô@ˆà*,°að 	ð 	ð 	ð 	ð 	õ ˜DœKÐ)?ÀÑDÔDˆ	ð Ø”K×1Ò1°&Ñ9Ô9Ø'+Ð3˜4œ8˜8°Ø06Ð>˜6œ;¨"Ñ,Ð,¸BØ28Ð@˜FœL¨BÑ.Ð.¸bØ06Ð>˜6œ;¨"Ñ,Ð,¸BØ#˜8 D¤KÔ$HÈ&ÐR[Ñ\Ô\Ø"˜( 4¤;Ô#FÈÐX\Ð]Ñ]Ô]Ø#˜8 D¤KÔ$OÐQWÐaeÐfÑfÔfð

ð 

ð 
	
r   c                ó(  — |                       ¦   «          | j        st          j        d¦  «         dS | j        }t	          d„ |D ¦   «         ¦  «        }dd›ddd|› �›dd	d
›ddd›ddd›ddd›�}t          j        d|› ddt          |¦  «        z  › �¦  «         |D ]Ð}|d         dk    r|d         d›d�nd}|d         dk    r|d         d
›d|d         d›�nd}|d         dk    r|d         › d�nd}|d         dk    r|d         d ›d|d!         d›�nd}t          j        |                     d"¦  «        d#›d|                     d$d%¦  «        d|› �›d|d
›d|d›d|d›d|d›�¦  «         ŒÑt          j        dt          |¦  «        z  › d�¦  «         dS )&z?Print GPU status in a compact table format using current stats.zNo GPU stats available.Nc              3  ó\   K  — | ]'}t          |                     d d¦  «        ¦  «        V — Œ(dS )r7   úN/AN)ÚlenÚget©r   rD   s     r   r    z'GPUInfo.print_status.<locals>.<genexpr>z   s8   è è € ÐDÐD°s•s˜3Ÿ7š7 6¨5Ñ1Ô1Ñ2Ô2ÐDÐDÐDÐDÐDÐDr   ÚIdxz<3ú ÚNameú<ÚUtilz>6z	Mem (MiB)z>15ÚTempz>5zPwr (W)z>10z
--- GPU Status ---
ú
ú-r8   r   ú%z N/A r9   ú/r:   z<6z N/A / N/A r<   ÚCr=   z>3r>   r%   z<3dr7   rQ   )r   r   r   r   ÚmaxÚinforR   rS   )	r   ÚstatsÚname_lenÚhdrrD   ÚuÚmÚtÚps	            r   Úprint_statuszGPUInfo.print_statusr   sF  € à×ÒÑÔÐØŒ~ð 	ÝŒNÐ4Ñ5Ô5Ð5ØˆFà”ˆÝÐDÐD¸eÐDÑDÔDÑDÔDˆØÐjÐjÐj˜FÐj XÐjÐjÐjÐj°ÐjÐjÐj¸[ÐjÐjÐjÈvÐjÐjÐjÐ[dÐjÐjÐjˆÝŒÐD¨SÐDÐD°C½#¸c¹(¼(±NÐDÐDÑEÔEÐEàð 	vð 	vˆCØ/2°=Ô/AÀQÒ/FÐ/F�3�}Ô%Ð+Ð+Ð+Ð+Ð+ÈGˆAØGJÈ=ÔGYÐ]^ÒG^ÐG^�3�}Ô%ÐCÐCÐC¨3¨~Ô+>ÐCÐCÐCÐCÐdqˆAØ,/°Ô,>À!Ò,CÐ,C�3�}Ô%Ð(Ð(Ð(Ð(ÈˆAØEHÈÔEVÐZ[ÒE[ÐE[�3�|Ô$ÐAÐAÐA¨#¨mÔ*<ÐAÐAÐAÐAÐahˆAåŒK˜3Ÿ7š7 7Ñ+Ô+ÐtÐtÐt°#·'²'¸&À%Ñ2HÔ2HÐtÈ8ÐtÐtÐtÐtÐWXÐtÐtÐtÐ^_ÐtÐtÐtÐfgÐtÐtÐtÐmnÐtÐtÐtÑuÔuÐuÐuåŒ�s�S ™XœX‘~Ð)Ð)Ð)Ñ*Ô*Ð*Ð*Ð*r   r+   r   NÚcountÚmin_memory_fractionr0   Úmin_util_fractionÚindicesúlist[int] | Noneú	list[int]c                ó|  ‡‡‡— ‰dk    sJ d‰› �¦   «         ‚‰dk    sJ d‰› �¦   «         ‚d‰dz  d›d‰dz  d›d�}t          j        d	|› d
|› d�¦  «         |dk    rg S |                      ¦   «          | j        st          j        d¦  «         g S ˆˆˆfd„| j        D ¦   «         }|                     d„ ¬¦  «         d„ |d|…         D ¦   «         }|rSt          |¦  «        |k     r(t          j        d|› dt          |¦  «        › d�¦  «         t          j        d|› �¦  «         nt          j        d|› d�¦  «         |S )aE  Select the most idle GPUs based on utilization and free memory.

        Args:
            count (int): The number of idle GPUs to select.
            min_memory_fraction (float): Minimum free memory required as a fraction of total memory.
            min_util_fraction (float): Minimum free utilization rate required from 0.0 - 1.0.
            indices (list[int] | None): Restrict selection to these GPU indices, e.g. the GPUs visible under an external
                CUDA_VISIBLE_DEVICES restriction. None searches all GPUs.

        Returns:
            (list[int]): Indices of the selected GPUs, sorted by idleness (lowest utilization first).

        Notes:
             Returns fewer than 'count' if not enough qualify or exist.
             Returns empty list if NVML stats are unavailable or no GPUs meet the criteria.
        g      ð?z(min_memory_fraction must be <= 1.0, got z&min_util_fraction must be <= 1.0, got zfree memory >= éd   z.1fz% and free utilization >= r]   zSearching for z idle GPUs with z...r   zNVML stats unavailable.c                óÎ   •— g | ]a}‰�
|d         ‰v rS|                      dd¦  «        |                      dd¦  «        z  ‰k    r"d|                      dd¦  «        z
  ‰dz  k    ¯_|‘ŒbS )Nr%   r;   r   r:   r+   rq   r8   )rS   )r   rD   rm   rk   rl   s     €€€r   ú
<listcomp>z+GPUInfo.select_idle_gpu.<locals>.<listcomp>¯   sŠ   ø€ ð 
ð 
ð 
àØ� 3 w¤<°7Ð#:Ð#:Ø—’˜ qÑ)Ô)¨C¯GªG°NÀAÑ,FÔ,FÑFÐJ]Ò]Ð]Ø�s—w’w˜}¨cÑ2Ô2Ñ2Ð7HÈ3Ñ7NÒNÐNð	 ð OÐNÐNr   c                ó€   — |                       dd¦  «        |                       dd¦  «         t          j        ¦   «         fS )Nr8   ée   r;   r   )rS   Úrandom)Úxs    r   ú<lambda>z)GPUInfo.select_idle_gpu.<locals>.<lambda>¸   s7   € ¨!¯%ª%°¸sÑ*CÔ*CÀaÇeÂeÈMÐ[\ÑF]ÔF]ÐE]Õ_eÔ_lÑ_nÔ_nÐ)o€ r   )Úkeyc                ó   — g | ]
}|d          ‘ŒS )r%   © rT   s     r   rs   z+GPUInfo.select_idle_gpu.<locals>.<listcomp>»   s   € ÐBÐBÐB S�C˜”LÐBÐBÐBr   Nz
Requested z GPUs but only z met the idle criteria.zSelected idle CUDA devices zNo GPUs met criteria (z).)r   ra   r   r   r   ÚsortrR   )r   rj   rk   rl   rm   ÚcriteriaÚeligible_gpusÚselecteds     ```   r   Úselect_idle_gpuzGPUInfo.select_idle_gpuˆ   sÒ  øøø€ ð. # cÒ)Ð)Ð)Ð+kÐViÐ+kÐ+kÑ)Ô)Ð)Ø  CÒ'Ð'Ð'Ð)eÐRcÐ)eÐ)eÑ'Ô'Ð'àuÐ1°CÑ7ÐuÐuÐuÐWhÐknÑWnÐuÐuÐuÐuð 	õ 	ŒÐI UÐIÐI¸HÐIÐIÐIÑJÔJÐJà�AŠ:ˆ:ØˆIà×ÒÑÔÐØŒ~ð 	ÝŒNÐ4Ñ5Ô5Ð5ØˆIð
ð 
ð 
ð 
ð 
ð 
à”~ð
ñ 
ô 
ˆð 	×ÒÐoÐoÐÑpÔpÐpð CÐB¨M¸&¸5¸&Ô,AÐBÑBÔBˆàð 	BÝ�8‰}Œ}˜uÒ$Ð$Ý”Ðh¨EÐhÐhÅ#ÀhÁ-Ä-ÐhÐhÐhÑiÔiÐiÝŒKÐ@°hÐ@Ð@ÑAÔAÐAÐAåŒNÐ@°HÐ@Ð@Ð@ÑAÔAÐAàˆr   )r%   r&   r'   r(   )r+   r   r   N)
rj   r&   rk   r0   rl   r0   rm   rn   r'   ro   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   r   r   ri   r€   r{   r   r   r   r      s¨   € € € € € ðð ðBTð Tð Tð ð ð ð(ð (ð (ð ð  ð  ð
ð 
ð 
ð 
ð6+ð +ð +ð0 Ø%&Ø#$Ø$(ð<ð <ð <ð <ð <ð <ð <r   r   Ú__main__gš™™™™™É?r+   )rj   rk   rl   z!
==> Using selected GPU indices: c                ó   — g | ]}d |› �‘ŒS )zcuda:r{   )r   Úidxs     r   rs   rs   Õ   s   € Ð5Ð5Ð5 S�=˜3�=�=Ð5Ð5Ð5r   z    Target devices: )Ú
__future__r   rv   Útypingr   Úultralytics.utilsr   r   r�   Úrequired_free_mem_fractionÚrequired_free_util_fractionÚnum_gpus_to_selectÚgpu_infori   r€   r   ÚprintÚdevicesr{   r   r   ú<module>r‘      s?  ðð #Ð "Ð "Ð "Ð "Ð "à €€€Ø Ð Ð Ð Ð Ð à $Ð $Ð $Ð $Ð $Ð $ðyð yð yð yð yñ yô yð yðx ˆzÒÐØ!$ÐØ"%ÐØÐàˆw‰yŒy€HØ×ÒÑÔÐà×+Ò+Ø Ø6Ø5ð ,ñ ô ð €xð 0ð
 	ˆÐ=°8Ð=Ð=Ñ>Ô>Ð>Ø5Ð5¨HÐ5Ñ5Ô5ˆØˆÐ. WÐ.Ð.Ñ/Ô/Ð/Ð/Ð/ð Ðð0ð 0r   