§
    “©ºjõ"  ã                   óÒ   — d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZ  e j        e¦  «        Z G d„ d	e¦  «        Zd
efd„Zd
efd„ZdS )é    N)ÚOptional)Úoverride)ÚClusterEnvironment)Ú_IS_WINDOWS)Úrank_zero_warn)ÚPossibleUserWarningc                   ól  ‡ — e Zd ZdZddedeej                 ddfˆ fd„Ze	e
defd„¦   «         ¦   «         Ze	e
defd	„¦   «         ¦   «         Ze	e
defd
„¦   «         ¦   «         Zee
defd„¦   «         ¦   «         Zedee         fd„¦   «         Zedee         fd„¦   «         Ze
defd„¦   «         Ze
deddfd„¦   «         Ze
defd„¦   «         Ze
deddfd„¦   «         Ze
defd„¦   «         Ze
defd„¦   «         Ze
dededdfd„¦   «         Zededefd„¦   «         Zedd„¦   «         Zedd„¦   «         Zˆ xZS )ÚSLURMEnvironmenta'  Cluster environment for training on a cluster managed by SLURM.

    You can configure the `main_address` and `main_port` properties via the env variables `MASTER_ADDR` and
    `MASTER_PORT`, respectively.

    Args:
        auto_requeue: Whether automatic job resubmission is enabled or not. How and under which conditions a job gets
            rescheduled gets determined by the owner of this plugin.
        requeue_signal: The signal that SLURM will send to indicate that the job should be requeued. Defaults to
            SIGUSR1 on Unix.

    TNÚauto_requeueÚrequeue_signalÚreturnc                 óÞ   •— t          ¦   «                              ¦   «          || _        |€t          st          j        }|| _        |                      ¦   «          |                      ¦   «          d S )N)	ÚsuperÚ__init__r   r   ÚsignalÚSIGUSR1r   Ú_validate_srun_usedÚ_validate_srun_variables)Úselfr   r   Ú	__class__s      €ú{/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/lightning_fabric/plugins/environments/slurm.pyr   zSLURMEnvironment.__init__/   se   ø€ Ý‰Œ×ÒÑÔÐØ(ˆÔØÐ!­+Ð!Ý#œ^ˆNØ,ˆÔØ× Ò Ñ"Ô"Ð"Ø×%Ò%Ñ'Ô'Ð'Ð'Ð'ó    c                 ó   — dS )NT© ©r   s    r   Úcreates_processes_externallyz-SLURMEnvironment.creates_processes_externally8   s	   € ð ˆtr   c                 ó*  — t           j                             d¦  «        }|€Dt           j                             dd¦  «        }|                      |¦  «        }|t           j        d<   t                               dt           j        d         › �¦  «         |S )NÚMASTER_ADDRÚSLURM_NODELISTz	127.0.0.1zMASTER_ADDR: )ÚosÚenvironÚgetÚresolve_root_node_addressÚlogÚdebug)r   Ú	root_nodeÚnodelists      r   Úmain_addresszSLURMEnvironment.main_address=   sz   € õ ”J—N’N =Ñ1Ô1ˆ	ØÐÝ”z—~’~Ð&6¸ÑDÔDˆHØ×6Ò6°xÑ@Ô@ˆIØ(1�BŒJ�}Ñ%å�	Š	Ð=¥"¤*¨]Ô";Ð=Ð=Ñ>Ô>Ð>ØÐr   c                 ót  — t           j                             d¦  «        }|�|dd …         }t          |¦  «        dz   }nd}dt           j        v r t          t           j        d         ¦  «        }nt	          |¦  «        t           j        d<   t
                               dt           j        d         › �¦  «         |S )NÚSLURM_JOB_IDéüÿÿÿi˜:  in2  ÚMASTER_PORTzMASTER_PORT: )r    r!   r"   ÚintÚstrr$   r%   )r   Újob_idÚdefault_ports      r   Ú	main_portzSLURMEnvironment.main_portI   s¤   € õ ”—’ Ñ/Ô/ˆØÐà! " # #œ;ˆLå˜|Ñ,Ô,¨uÑ4ˆLˆLà ˆLð �BœJÐ&Ð&Ý�rœz¨-Ô8Ñ9Ô9ˆLˆLå(+¨LÑ(9Ô(9�BŒJ�}Ñ%å�	Š	Ð=¥"¤*¨]Ô";Ð=Ð=Ñ>Ô>Ð>ØÐr   c                  óP   — t                                ¦   «          t          ¦   «         S )aÒ  Returns ``True`` if the current process was launched on a SLURM cluster.

        It is possible to use the SLURM scheduler to request resources and then launch processes manually using a
        different environment. For this, the user can set the job name in SLURM to 'bash' or 'interactive' (srun --job-
        name=interactive). This will then avoid the detection of ``SLURMEnvironment`` and another environment can be
        detected automatically.

        )r
   r   Ú_is_srun_usedr   r   r   ÚdetectzSLURMEnvironment.detecte   s    € õ 	×,Ò,Ñ.Ô.Ð.Ý‰ŒÐr   c                  ó@   — t           j                             d¦  «        S )NÚSLURM_JOB_NAME)r    r!   r"   r   r   r   Újob_namezSLURMEnvironment.job_names   s   € åŒz�~Š~Ð.Ñ/Ô/Ð/r   c                  óª   — t          ¦   «         rd S t          j                             d¦  «        } | €d S 	 t	          | ¦  «        S # t
          $ r Y d S w xY w)Nr*   ©Ú_is_slurm_interactive_moder    r!   r"   r-   Ú
ValueError)r/   s    r   r/   zSLURMEnvironment.job_idw   sg   € õ &Ñ'Ô'ð 	Ø�4å”—’ Ñ/Ô/ˆØˆ>Ø�4ð	Ý�v‘;”;ÐøÝð 	ð 	ð 	Ø�4�4ð	øøøs   µA Á
AÁAc                 ó@   — t          t          j        d         ¦  «        S ©NÚSLURM_NTASKS©r-   r    r!   r   s    r   Ú
world_sizezSLURMEnvironment.world_size…   ó   € å•2”:˜nÔ-Ñ.Ô.Ð.r   Úsizec                 ó:   — t                                d¦  «         d S )Nz[SLURMEnvironment.set_world_size was called, but setting world size is not allowed. Ignored.©r$   r%   )r   rB   s     r   Úset_world_sizezSLURMEnvironment.set_world_size‰   s   € å�	Š	ÐoÑpÔpÐpÐpÐpr   c                 ó@   — t          t          j        d         ¦  «        S )NÚSLURM_PROCIDr?   r   s    r   Úglobal_rankzSLURMEnvironment.global_rank�   rA   r   Úrankc                 ó:   — t                                d¦  «         d S )Nz]SLURMEnvironment.set_global_rank was called, but setting global rank is not allowed. Ignored.rD   )r   rI   s     r   Úset_global_rankz SLURMEnvironment.set_global_rank‘   s   € å�	Š	ÐqÑrÔrÐrÐrÐrr   c                 ó@   — t          t          j        d         ¦  «        S )NÚSLURM_LOCALIDr?   r   s    r   Ú
local_rankzSLURMEnvironment.local_rank•   s   € å•2”:˜oÔ.Ñ/Ô/Ð/r   c                 ó@   — t          t          j        d         ¦  «        S )NÚSLURM_NODEIDr?   r   s    r   Ú	node_rankzSLURMEnvironment.node_rank™   rA   r   Únum_devicesÚ	num_nodesc           	      ó^  — t          ¦   «         rd S t          j                             d¦  «        }|�,t	          |¦  «        |k    rt          d|› d|› d|› d�¦  «        ‚t          j                             d¦  «        }|�,t	          |¦  «        |k    rt          d|› d|› d	|› d�¦  «        ‚d S d S )
NÚSLURM_NTASKS_PER_NODEzYou set `devices=zX` in Lightning, but the number of tasks per node configured in SLURM `--ntasks-per-node=z%` does not match. HINT: Set `devices=z`.ÚSLURM_NNODESzYou set `num_nodes=zE` in Lightning, but the number of nodes configured in SLURM `--nodes=z'` does not match. HINT: Set `num_nodes=r9   )r   rR   rS   Úntasks_per_nodeÚnnodess        r   Úvalidate_settingsz"SLURMEnvironment.validate_settings�   s  € å%Ñ'Ô'ð 	ØˆFÝœ*Ÿ.š.Ð)@ÑAÔAˆØÐ&­3¨Ñ+?Ô+?À;Ò+NÐ+NÝðq Kð qð qØ'6ðqð qØ]lðqð qð qñô ð õ ”—’ Ñ/Ô/ˆØÐ¥# f¡+¤+°Ò":Ð":ÝðW ið Wð WØ#ðWð WØLRðWð Wð Wñô ð ð ÐÐ":Ð":r   Únodesc                 óÂ   — t          j        dd| ¦  «        } t          j        dd| ¦  «        } |                      d¦  «        d                              d¦  «        d         S )a¢  The node selection format in SLURM supports several formats.

        This function selects the first host name from

        - a space-separated list of host names, e.g., 'host0 host1 host3' yields 'host0' as the root
        - a comma-separated list of host names, e.g., 'host0,host1,host3' yields 'host0' as the root
        - the range notation with brackets, e.g., 'host[5-9]' yields 'host5' as the root

        z\[(.*?)[,-].*\]z\1z	\[(.*?)\]ú r   ú,)ÚreÚsubÚsplit)rZ   s    r   r#   z*SLURMEnvironment.resolve_root_node_address®   sU   € õ ”Ð)¨5°%Ñ8Ô8ˆÝ”�| U¨EÑ2Ô2ˆØ�{Š{˜3ÑÔ Ô"×(Ò(¨Ñ-Ô-¨aÔ0Ð0r   c                  óD  — t           rdS t          j        d¦  «        du} | r|t          ¦   «         spd                     dt
          j                             t          j	        ¦  «        gt          j
        ¢¦  «        dd…         }t          d|› d�t          ¬¦  «         dS dS dS )aU  Checks if the `srun` command is available and used.

        Parallel jobs (multi-GPU, multi-node) in SLURM are launched by prepending `srun` in front of the Python command.
        Not doing so will result in processes hanging, which is a frequent user error. Lightning will emit a warning if
        `srun` is found but not used.

        NÚsrunr\   é@   z§The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: z ...)Úcategory)r   ÚshutilÚwhichr3   Újoinr    ÚpathÚbasenameÚsysÚ
executableÚargvr   r   )Úsrun_existsÚhints     r   r   z$SLURMEnvironment._validate_srun_used½   sÆ   € õ ð 	ØˆFå”l 6Ñ*Ô*°$Ð6ˆØð 	�}™œð 	Ø—8’8˜V¥R¤W×%5Ò%5µc´nÑ%EÔ%EÐQÍÌÐQÑRÔRÐSVÐTVÐSVÔWˆDÝðdØY]ðdð dð då,ðñ ô ð ð ð ð	ð 	ð 	ð 	r   c                  ó¸   — t          t          j                             dd¦  «        ¦  «        } | dk    r$dt          j        vrt	          d| › d| › d�¦  «        ‚dS dS )	a2  Checks for conflicting or incorrectly set variables set through `srun` and raises a useful error message.

        Right now, we only check for the most common user errors. See
        `the srun docs <https://slurm.schedmd.com/srun.html>`_
        for a complete list of supported srun variables.

        r>   Ú1é   rU   zYou set `--ntasks=z^` in your SLURM bash script, but this variable is not supported. HINT: Use `--ntasks-per-node=z
` instead.N)r-   r    r!   r"   ÚRuntimeError)Úntaskss    r   r   z)SLURMEnvironment._validate_srun_variablesÒ   s   € õ •R”Z—^’^ N°CÑ8Ô8Ñ9Ô9ˆØ�AŠ:ˆ:Ð1½¼ÐCÐCÝðD Vð Dð DØ17ðDð Dð Dñô ð ð ˆ:ÐCÐCr   )TN)r   N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úboolr   r   ÚSignalsr   Úpropertyr   r   r.   r(   r-   r1   Ústaticmethodr4   r7   r/   r@   rE   rH   rK   rN   rQ   rY   r#   r   r   Ú__classcell__)r   s   @r   r
   r
   !   s*  ø€ € € € € ðð ð(ð ( Tð (À(È6Ì>ÔBZð (Ðfjð (ð (ð (ð (ð (ð (ð Øð¨dð ð ð ñ „Xñ „Xðð Øð˜cð ð ð ñ „Xñ „Xðð Øð˜3ð ð ð ñ „Xñ „Xðð4 Øð
�Dð 
ð 
ð 
ñ „Xñ „\ð
ð ð0�h˜s”mð 0ð 0ð 0ñ „\ð0ð ð�H˜S”Mð ð ð ñ „\ðð ð/˜Cð /ð /ð /ñ „Xð/ð ðq 3ð q¨4ð qð qð qñ „Xðqð ð/˜Sð /ð /ð /ñ „Xð/ð ðs Cð s¨Dð sð sð sñ „Xðsð ð0˜Cð 0ð 0ð 0ñ „Xð0ð ð/˜3ð /ð /ð /ñ „Xð/ð ð¨Sð ¸Sð ÀTð ð ð ñ „Xðð  ð1¨ð 1°ð 1ð 1ð 1ñ „\ð1ð ðð ð ñ „\ðð( ðð ð ñ „\ðð ð ð ð r   r
   r   c                  ó<   — dt           j        v ot          ¦   «          S r=   )r    r!   r:   r   r   r   r3   r3   ã   s   € Ø�RœZÐ'ÐLÕ0JÑ0LÔ0LÐ,LÐLr   c                  ó8   — t                                ¦   «         dv S )N)ÚbashÚinteractive)r
   r7   r   r   r   r:   r:   ç   s   € Ý×$Ò$Ñ&Ô&Ð*AÐAÐAr   )Úloggingr    r^   re   r   rj   Útypingr   Útyping_extensionsr   Ú9lightning_fabric.plugins.environments.cluster_environmentr   Ú"lightning_fabric.utilities.importsr   Ú$lightning_fabric.utilities.rank_zeror   Ú#lightning_fabric.utilities.warningsr   Ú	getLoggerrt   r$   r
   rx   r3   r:   r   r   r   ú<module>r‰      s;  ðð €€€Ø 	€	€	€	Ø 	€	€	€	Ø €€€Ø €€€Ø 
€
€
€
Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &à XÐ XÐ XÐ XÐ XÐ XØ :Ð :Ð :Ð :Ð :Ð :Ø ?Ð ?Ð ?Ð ?Ð ?Ð ?Ø CÐ CÐ CÐ CÐ CÐ Cà€gÔ˜Ñ!Ô!€ðð ð ð ð Ð)ñ ô ð ðDM�tð Mð Mð Mð MðB Dð Bð Bð Bð Bð Bð Br   