§
    ­¸j™  ã                  ó`  — d Z ddlmZ ddlZddlmZ ddlmZmZ ddl	Z	ddl
mZmZmZmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ  e¦   «         Z ed¬¦  «         G d„ d¦  «        ¦   «         ZdQd„Z	 	 	 dRdSd"„ZdTd%„Z	 	 dUdVd0„Zdddddd1œdWd=„Z	 	 	 	 	 	 	 	 	 	 dXdYdF„Z dZdI„Z!	 	 d[d\dP„Z"dS )]a%  Auto-batch probing: find a safe micro-batch size before training.

Probe assumptions (worst-case so training does not OOM):
- Resolution: When multi_scale is True we use the maximum of the multi-scale
  augmentation scales (same as compute_multi_scale_scales). Otherwise we use model resolution. This ensures the step
  uses the max resolution seen in training.
- Targets: Memory grows with number of targets per image. We use
  auto_batch_max_targets_per_image (config) to synthesize that many targets per image so the probe reflects worst-case
  matcher and loss memory.
- EMA: When use_ema is True, an EMA copy of the model is kept in memory. We
  apply auto_batch_ema_headroom (e.g. 0.7) to the probed batch size so the effective safe batch leaves room for the EMA
  model.
- Optimizer state: AdamW allocates exp_avg/exp_avg_sq lazily, on its first
  step() rather than at construction, each the same size as the trainable parameters -- so the combined state is 2x
  that size, 3x with amsgrad. The probe runs a shadow AdamW step (see _build_shadow_optimizer) so that memory is part
  of the estimate instead of first appearing on training's own first optimizer step. Only AdamW-shaped optimizers are
  modeled this way (see _is_adamw_shaped); anything else gets the runtime warning in resolve_auto_batch_config.
é    )ÚannotationsN)Ú	dataclass)ÚAnyÚcast)ÚModelConfigÚ
MultiScaleÚTrainConfigÚ_resolve_amp_dtype)Úcompute_multi_scale_scales)Úbuild_criterion_from_config)Ú_is_builtin_fused_adamw)Ú
get_logger)ÚNestedTensorT)Úfrozenc                  ó<   — e Zd ZU dZded<   ded<   ded<   ded<   dS )	ÚAutoBatchResulta­  Result of auto-batch probing: safe micro-batch size and recommended grad accumulation.

    Attributes:
        safe_micro_batch: Per-device batch size that fits in memory for one train step.
        recommended_grad_accum_steps: Steps to accumulate to reach target effective batch.
        effective_batch_size: safe_micro_batch * recommended_grad_accum_steps.
        device_name: Human-readable GPU name used for probing.
    ÚintÚsafe_micro_batchÚrecommended_grad_accum_stepsÚeffective_batch_sizeÚstrÚdevice_nameN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú__annotations__© ó    új/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/rfdetr/training/auto_batch.pyr   r   +   sN   € € € € € € ðð ð ÐÐÑØ%Ð%Ð%Ñ%ØÐÐÑØÐÐÑÐÐr   r   ÚexcÚBaseExceptionÚreturnÚboolc                óT   — t          | ¦  «                             ¦   «         }d|v pd|v S )Nzout of memoryzcuda error: out of memory)r   Úlower)r!   Úmessages     r    Ú_is_cuda_oomr(   <   s-   € Ý�#‰hŒh�nŠnÑÔ€GØ˜gÐ%ÐOÐ)DÈÐ)OÐOr   Fé   é   Úmicro_batch_sizer   Ú
resolutionÚdeviceútorch.deviceÚnum_classesÚsegmentation_headÚmax_targets_per_imageÚnum_channelsú2tuple[NestedTensor, list[dict[str, torch.Tensor]]]c           
     óâ  — t          j        | ||||¬¦  «        }t          j        | ||t           j        |¬¦  «        }t	          ||¦  «        }	t          d|dz
  ¦  «        }
t          d|¦  «        }g }t          | ¦  «        D �]j}t          j        g d¢gt           j        |¬¦  «         	                    |d¦  «        }t          j        t          d|
¦  «        gt           j        |¬¦  «         	                    |¦  «        }t          j        |t           j        |¬¦  «        }t          j        |fdt           j        |¬¦  «        }||t          j        |t           j        |¬¦  «        t          j        ||gt           j        |¬¦  «        t          j        ||gt           j        |¬¦  «        ||dœ}|r&t          j        |||t           j        |¬¦  «        |d	<   |                     |¦  «         �Œl|	|fS )
a  Build a minimal (samples, targets) batch for probing.

    Uses max_targets_per_image targets per image so memory reflects worst-case matcher and loss. When segmentation_head
    is True, each target dict includes "masks" of shape (max_targets_per_image, resolution, resolution).
    )r-   )Údtyper-   r   r)   )ç      à?r6   çš™™™™™É?r7   é   g{®Gáz¤?)ÚboxesÚlabelsÚimage_idÚ	orig_sizeÚsizeÚiscrowdÚareaÚmasks)ÚtorchÚrandnÚzerosr$   r   ÚmaxÚrangeÚtensorÚfloat32ÚexpandÚminÚint64ÚfullÚappend)r+   r,   r-   r/   r0   r1   r2   ÚtensorsÚmaskÚsamplesÚ	max_labelÚnÚtargetsÚidxr9   r:   r>   r?   Úts                      r    Ú_make_synthetic_batchrU   A   sÜ  € õ ŒkÐ*¨L¸*ÀjÐY_Ð`Ñ`Ô`€GÝŒ;Ð'¨°ZÅuÄzÐZ`ÐaÑaÔa€DÝ˜7 DÑ)Ô)€Gå�A�{ Q‘Ñ'Ô'€IÝˆAÐ$Ñ%Ô%€AØ-/€GÝÐ%Ñ&Ô&ð ñ ˆå”Ð2Ð2Ð2Ð3½5¼=ÐQWÐXÑXÔX×_Ò_Ð`aÐcdÑeÔeˆÝ”�s 1 iÑ0Ô0Ð1½¼ÈVÐTÑTÔT×[Ò[Ð\]Ñ^Ô^ˆÝ”+˜a¥u¤{¸6ÐBÑBÔBˆÝŒz˜1˜$ ­E¬MÀ&ÐIÑIÔIˆàØÝœ Sµ´ÀFÐKÑKÔKÝœ z°:Ð&>ÅeÄkÐZ`ÐaÑaÔaÝ”L *¨jÐ!9ÅÄÐU[Ð\Ñ\Ô\ØØð&
ð &
ˆð ð 	aÝœ Q¨
°JÅeÄjÐY_Ð`Ñ`Ô`ˆAˆg‰JØ�Š�qÑÔÐÑØ�GÐÐr   Úoptimizer_cfgÚobjectc                ó˜   — t          | t          ¦  «        sdS |                      ¦   «         }d|v r|dv S |                     ¦   «         dk    S )a  Return whether a configured optimizer allocates the same per-parameter state as ``torch.optim.AdamW``.

    Deliberately broader than ``module_model._is_builtin_fused_adamw``, which answers a different question (does this
    config select RF-DETR's managed fused-kernel path?) and must stay narrow. For a memory estimate the only thing that
    matters is whether the real optimizer keeps ``exp_avg``/``exp_avg_sq`` per parameter, which a dotted path importing
    ``torch.optim.AdamW`` does just as much as the built-in ``"adamw"`` short name.

    Args:
        optimizer_cfg: The ``TrainConfig.optimizer`` value; a callable's state layout is unknowable here, so it
            reports ``False``.

    Returns:
        ``True`` for the ``"adamw"`` short name (case-insensitive) and for a dotted path naming ``torch.optim.AdamW``.

    Examples:
        >>> _is_adamw_shaped("AdamW")
        True
        >>> _is_adamw_shaped("torch.optim.AdamW")
        True
        >>> _is_adamw_shaped("torch.optim.SGD")
        False
    Fú.>   útorch.optim.AdamWútorch.optim.adamw.AdamWÚadamw)Ú
isinstancer   Ústripr&   )rV   Únames     r    Ú_is_adamw_shapedr`   k   sV   € õ. �m¥SÑ)Ô)ð ØˆuØ×ÒÑ Ô €DØ
ˆd€{€{ð ÐGÐGÐGØ�:Š:‰<Œ<˜7Ò"Ð"r   Útrainable_paramsúlist[torch.nn.Parameter]ÚlrÚfloatÚweight_decayÚoptimizer_kwargsúdict[str, Any] | NoneÚfusedúbool | Noneú0tuple[torch.optim.Optimizer, list[torch.Tensor]]c                óÚ   — d„ | D ¦   «         }|||dœ}|                      |pi ¦  «         	 t          j        j        |fi |¤Ž}n&# t          $ r}t	          d|› d�¦  «        |‚d}~ww xY w||fS )a^  Build an AdamW optimizer over throwaway tensors shaped like ``trainable_params``, for probing optimizer-state
    memory without ever writing to the real model's weights.

    ``optimizer.step()`` needs to run against *some* set of parameters to allocate
    AdamW's per-parameter state (``exp_avg``/``exp_avg_sq``), but running it against the real model's own
    parameters would apply a real (garbage, synthetic-data-derived) gradient update to weights that
    may already hold a loaded pretrained checkpoint -- corrupting them before training even starts.
    The shadow tensors this returns are separate GPU allocations of the same shape/dtype, so the
    optimizer state they accumulate costs the same memory as the real optimizer eventually would,
    without the update ever touching a real parameter.

    That separateness is also what this design costs: the shadow copies are allocated alongside the
    model's own already-resident parameters, so the probe holds one extra copy of the trainable
    parameters that real training never holds. On the non-fused path (amp off, fp16, or
    ``fused_optimizer=False``) there is a second, transient divergence: the shadow optimizer steps a
    single parameter group, so ``_foreach_sqrt`` builds one whole-model-sized temporary, where real
    training's per-layer-group parameter dicts (``get_param_dict``, ``module_model.py``) keep that
    temporary per-group. Both make the probe stricter than the run it sizes rather than more
    permissive, and the transient one does not arise at all whenever the probe runs fused -- the
    common bf16-on-CUDA default.

    Args:
        trainable_params: The real model parameters being probed for (only used for shape/dtype/device).
        lr: Learning rate for the shadow optimizer (does not affect state tensor size).
        weight_decay: Weight decay for the shadow optimizer (does not affect state tensor size).
        optimizer_kwargs: Extra AdamW kwargs from ``train_config.optimizer_kwargs`` (e.g. ``amsgrad=True``,
            which adds a third ``max_exp_avg_sq`` state buffer). Only meaningful when the real training run
            uses an AdamW-shaped optimizer (see ``_is_adamw_shaped``) -- pass ``None`` otherwise, since these
            kwargs are AdamW-specific and would raise for a different optimizer class. Keys given here override
            ``lr``/``weight_decay``/``fused`` below: on the dotted-path config (``"torch.optim.AdamW"``)
            ``configure_optimizers`` builds the real optimizer from this dict alone, injecting no lr or
            weight_decay of its own (``module_model.py``), so this dict is where that run's real values live.
        fused: Whether to build a fused AdamW, matching ``RFDETRModelModule._use_fused_optimizer`` for the real
            optimizer (``module_model.py``). The fused and foreach/single-tensor implementations differ in
            step-time temporary memory, so leaving this unset (``None``, torch's own default selection) can
            make the probe's memory profile diverge from the real optimizer's -- in either direction.

    Returns:
        The shadow optimizer and the list of shadow tensors it steps, in the same order as
        ``trainable_params`` so gradients can be paired up by index before each ``step()``.

    Raises:
        TypeError: If ``optimizer_kwargs`` carries an argument ``torch.optim.AdamW`` does not accept.
    c                ó6   — g | ]}t          j        |¦  «        ‘ŒS r   )rA   Ú
zeros_like©Ú.0Úps     r    ú
<listcomp>z+_build_shadow_optimizer.<locals>.<listcomp>Á   s#   € ÐCÐCÐC¨Q•UÔ% aÑ(Ô(ÐCÐCÐCr   )rc   re   rh   zFFailed to initialize the auto-batch probe's shadow 'adamw' optimizer: zF. Check optimizer_kwargs for arguments supported by torch.optim.AdamW.N)ÚupdaterA   ÚoptimÚAdamWÚ	TypeError)	ra   rc   re   rf   rh   Úshadow_paramsÚadamw_kwargsÚ	optimizerr!   s	            r    Ú_build_shadow_optimizerry   Œ   sÅ   € ðj DÐCÐ2BÐCÑCÔC€Mð +-¸lÐUZÐ#[Ð#[€LØ×ÒÐ(Ð.¨BÑ/Ô/Ð/ðÝ”KÔ% mÐDÐD°|ÐDÐDˆ	ˆ	øÝð ð ð ÝðSÐUXð Sð Sð Sñ
ô 
ð ð	øøøøðøøøð
 �mÐ#Ð#s   «A Á
A&ÁA!Á!A&)r0   r1   r2   Úautocast_dtypeÚwarn_missing_gradsÚmodelútorch.nn.ModuleÚ	criterionÚampÚshadow_optimizerútorch.optim.Optimizerrv   úlist[torch.Tensor]rz   útorch.dtype | Noner{   c          	     ó¨  ‡‡— 	 |                       d¬¦  «         |                      d¬¦  «         t          |||||
||¬¦  «        \  }}d|dœ}|�||d<   t          j        di |¤Ž5   | ||¦  «        }t	          d |||¦  «        ¦  «        Št	          d	|j        ¦  «        Šˆˆfd
„‰D ¦   «         }|st          d¦  «        ‚t          j        |¦  «                             ¦   «         }ddd¦  «         n# 1 swxY w Y   t          j	        |¦  «        st          d¦  «        ‚t          j
                             |¦  «         t          |	|d¬¦  «        D ]\  }}|j        |_        Œ|rt          d„ |D ¦   «         ¦  «        nd}	 |                     ¦   «          |	D ]	}d|_        Œ
n# |	D ]	}d|_        Œ
w xY w|                       d¬¦  «         |                      d¬¦  «         |r)t                               d|t#          |¦  «        ¦  «         dS # t          $ r9}t%          |¦  «        r$t          j                             ¦   «          Y d}~dS ‚ d}~ww xY w)að  Run one forward + loss + backward + shadow optimizer step; return True if successful, False on OOM.

    The shadow step matters for the memory estimate, not just for realism: AdamW (and most stateful optimizers) allocate
    their per-parameter state (``exp_avg``/``exp_avg_sq``, each the same size as the parameter itself) lazily, on the
    first ``step()`` call -- forward+backward alone never touches that allocation.
    ``shadow_optimizer``/``shadow_params`` are built once by the caller and reused across every probe iteration, so this
    first call is the only one that pays the allocation; every later iteration (larger candidate batch sizes) competes
    for GPU memory against that already-resident state, the same way a real second-and-later training step would. The
    shadow tensors, not the real model parameters, receive the update, so this never mutates the model being probed.

    ``warn_missing_grads`` asks this step to report parameters the synthetic batch never reached: AdamW allocates state
    only for parameters that carry a gradient, so those are absent from the estimate. The caller sets it for the first
    step only (every later iteration would recount the same parameters), and the report is emitted only if that step
    goes on to succeed.

    Args:
        model: The model to probe; already in train mode, and left untouched apart from its gradients.
        criterion: The loss criterion (must match model output and target format).
        micro_batch_size: Per-device batch size to attempt in this single step.
        resolution: Input spatial size (square) for the synthetic images.
        device: CUDA device the synthetic batch is allocated on.
        num_classes: Number of classes, used to pick synthetic target labels.
        amp: Whether to run the forward under autocast.
        trainable_params: The model's ``requires_grad`` parameters, in the order ``shadow_params`` was built from --
            index i of one must correspond to index i of the other, which the aliasing loop asserts.
        shadow_optimizer: The AdamW built over ``shadow_params``; stepped here so its state is allocated and stays
            resident for later iterations.
        shadow_params: Throwaway tensors mirroring ``trainable_params``, which take the update in their place.
        segmentation_head: If True, synthetic targets include "masks" so loss_masks is exercised.
        max_targets_per_image: Number of synthetic targets per image (worst-case matcher and loss memory).
        num_channels: Number of input image channels for the synthetic images.
        autocast_dtype: Explicit autocast dtype; ``None`` leaves torch's own default for the device.
        warn_missing_grads: Whether to report parameters that received no gradient (see above). Off by default so
            only the caller's chosen step reports.

    Returns:
        True if the step ran to completion, False if it hit a CUDA OOM (the signal that this ``micro_batch_size``
        does not fit). Any other RuntimeError propagates.
    T©Úset_to_none)r+   r,   r-   r/   r0   r1   r2   Úcuda)Údevice_typeÚenabledNr5   zdict[str, torch.Tensor]zdict[str, float]c                ó:   •— g | ]}|‰v ¯‰|         ‰|         z  ‘ŒS r   r   )ro   r_   Ú	loss_dictÚweight_dicts     €€r    rq   z_probe_step.<locals>.<listcomp>  s3   ø€ ÐpÐpÐpÀtÐ\`ÐdoÐ\oÐ\o˜y¨œ°¸TÔ1BÑBÐ\oÐ\oÐ\or   znauto-batch probe could not build weighted losses: no overlap between criterion loss_dict and weight_dict keys.z5auto-batch probe produced a non-finite training loss.)Ústrictc              3  ó(   K  — | ]}|j         ­	dV — Œd S )Nr)   )Úgrad)ro   Úreal_ps     r    ú	<genexpr>z_probe_step.<locals>.<genexpr>/  s)   è è € ÐTÐT &ÀÄÐ@S˜AÐ@SÐ@SÐ@SÐ@SÐTÐTr   r   z°[auto-batch] %d/%d trainable params received no gradient from the synthetic probe batch; their optimizer state is not modeled, so the returned batch size may be too permissive.Fr   )Ú	zero_gradrU   rA   Úautocastr   rŒ   ÚRuntimeErrorÚstackÚsumÚisfiniteÚautogradÚbackwardÚzipr�   ÚstepÚloggerÚwarningÚlenr(   r‡   Úempty_cache)r|   r~   r+   r,   r-   r/   r   ra   r€   rv   r0   r1   r2   rz   r{   rO   rR   Úautocast_kwargsÚoutputsÚweighted_lossesÚlossÚshadow_pr�   Úmissing_gradsr!   r‹   rŒ   s                            @@r    Ú_probe_stepr¦   Ñ   s'  øø€ ðr>Ø�Š DˆÑ)Ô)Ð)Ø×Ò¨ÐÑ-Ô-Ð-Ý0Ø-Ø!ØØ#Ø/Ø"7Ø%ð
ñ 
ô 
Ñˆ�ð ;AÈSÐ*QÐ*QˆØÐ%Ø'5ˆO˜GÑ$ÝŒ^Ð.Ð.˜oÐ.Ð.ð 
	6ð 
	6Ø�e˜G WÑ-Ô-ˆGÝÐ6¸	¸	À'È7Ñ8SÔ8SÑTÔTˆIÝÐ1°9Ô3HÑIÔIˆKØpÐpÐpÐpÐpÈyÐpÑpÔpˆOØ"ð Ý"ð(ñô ð õ ”;˜Ñ/Ô/×3Ò3Ñ5Ô5ˆDð
	6ð 
	6ð 
	6ñ 
	6ô 
	6ð 
	6ð 
	6ð 
	6ð 
	6ð 
	6ð 
	6øøøð 
	6ð 
	6ð 
	6ð 
	6õ Œ~˜dÑ#Ô#ð 	XÝÐVÑWÔWÐWåŒ×Ò Ñ%Ô%Ð%õ !$ MÐ3CÈDÐ QÑ QÔ Qð 	(ð 	(ÑˆH�fØ"œKˆHŒMˆMØXjÐq�ÐTÐTÐ,<ÐTÑTÔTÑTÔTÐTÐpqˆð		%Ø×!Ò!Ñ#Ô#Ð#ð *ð %ð %�Ø $�”�ð%ø˜Mð %ð %�Ø $�”�ð%øøøà�Š DˆÑ)Ô)Ð)Ø×Ò¨ÐÑ-Ô-Ð-Øð 	Ý�NŠNðdàÝÐ$Ñ%Ô%ñ	ô ð ð ˆtøÝð ð ð Ý˜ÑÔð 	ÝŒJ×"Ò"Ñ$Ô$Ð$Ø�5�5�5�5�5Øøøøøð	øøøs]   „A"H Á&BC4Ã(H Ã4C8Ã8H Ã;C8Ã<BH ÆF$ ÆH Æ$F2Æ2AH È
IÈ-IÉIÉIçÍÌÌÌÌÌì?é€   ç-Cëâ6?Úsafety_marginÚmax_micro_batchÚoptimizer_lrÚoptimizer_weight_decayÚoptimizer_fusedc                óØ  ‡ ‡‡‡‡‡‡‡‡
‡‡‡‡‡— ‰j         dk    rt          d¦  «        ‚|	dk     rt          d¦  «        ‚d|cxk     rdk    sn t          d¦  «        ‚‰ j        }‰j        }dŠdŠ	 ‰                      ¦   «          ‰                     ¦   «          d	„ ‰                      ¦   «         D ¦   «         Š‰st          d
¦  «        ‚	 t          ‰||||¦  «        \  ŠŠn2# t          $ r%}t          |¦  «        s‚ t          d¦  «        |‚d}~ww xY wdŠdˆˆˆˆˆˆ ˆ
ˆˆˆˆˆˆˆfd„} |d¦  «        st          d¦  «        ‚ |d¦  «        st          d¦  «        ‚d}d}d}||	k    r ||¦  «        r|}|dz  }n|}n||	k    °|€|	dz   }|dz   }t          |dz
  |	¦  «        }||k    r&||z   dz  } ||¦  «        r|}|dz   }n|dz
  }||k    °&t          dt          j        ||z  ¦  «        ¦  «        }|‰                      |¦  «         ‰                     |¦  «         ‰                      d¬¦  «         ‰                     d¬¦  «         ‰�‹‰�‹t          j                             ¦   «          S # ‰                      |¦  «         ‰                     |¦  «         ‰                      d¬¦  «         ‰                     d¬¦  «         ‰�‹‰�‹t          j                             ¦   «          w xY w)a
  Find the largest per-device batch size that fits in memory for one train step.

    Uses exponential search (1, 2, 4, ...) up to the first failure, then binary search between the last successful size
    and the first failure to get the exact maximum. The returned value is floor(max_ok * safety_margin), so
    safety_margin in (0, 1] scales down the result for headroom (e.g. 0.9 keeps 10% margin).

    Every probe iteration also runs a shadow AdamW step (see ``_build_shadow_optimizer``): forward and
    backward alone never trigger the ``exp_avg``/``exp_avg_sq`` state AdamW allocates lazily on its
    first real ``step()`` -- each buffer the same size as the trainable parameters themselves, so the
    combined state is 2x that size (3x with ``amsgrad``) -- so a probe that skipped this would report a
    batch size that fits forward+backward but can still OOM training on its first real optimizer step.

    Args:
        model: The model to probe (will be set to train mode).
        criterion: The loss criterion (must match model output and target format).
        resolution: Input spatial size (square).
        device: CUDA device to run on.
        num_classes: Number of classes (for synthetic targets).
        amp: Whether to use autocast for the forward.
        segmentation_head: If True, synthetic targets include "masks" for loss_masks.
        max_targets_per_image: Number of synthetic targets per image (worst-case memory).
        safety_margin: Fraction of max batch to return (0 < safety_margin <= 1).
        max_micro_batch: Cap on batch size to try.
        num_channels: Number of input image channels (for synthetic probe images).
        optimizer_lr: Learning rate for the shadow AdamW optimizer used to size its state (does not
            affect the state tensors' shape/size, only included so the shadow optimizer is
            constructible without a training config on hand). Not derived from ``train_config``:
            ``resolve_auto_batch_config`` leaves this and ``optimizer_weight_decay`` at their defaults,
            since neither changes the memory the probe measures.
        optimizer_weight_decay: Weight decay for the shadow AdamW optimizer (same caveat as ``optimizer_lr``).
        optimizer_kwargs: Extra AdamW kwargs from ``train_config.optimizer_kwargs`` (e.g. ``amsgrad=True``).
            Pass only when the real run uses the built-in ``"adamw"`` path -- see ``_build_shadow_optimizer``.
        optimizer_fused: Whether the real training run will use fused AdamW (mirrors
            ``RFDETRModelModule._use_fused_optimizer``). Forwarded to ``_build_shadow_optimizer`` so the shadow
            optimizer's step-time temporary memory matches the real one's, since fused and foreach/single-tensor
            AdamW allocate different amounts of scratch memory during ``step()``.

    Returns:
        Safe micro-batch size (>= 1).

    Raises:
        RuntimeError: If device is not CUDA or if micro_batch_size=1 already fails (OOM).
        ValueError: If max_micro_batch < 1 or safety_margin not in (0, 1].
    r‡   z0auto-batch probing currently supports CUDA only.r)   zmax_micro_batch must be >= 1.r   g      ð?z safety_margin must be in (0, 1].Nc                ó    — g | ]}|j         ¯	|‘ŒS r   )Úrequires_gradrn   s     r    rq   z)probe_max_micro_batch.<locals>.<listcomp>™  s   € ÐMÐMÐM !¸Q¼_ÐM˜AÐMÐMÐMr   zlauto-batch probing requires at least one trainable parameter; every model parameter has requires_grad=False.a  auto-batch probe ran out of memory allocating the shadow optimizer's parameter copies, before any micro_batch_size was tried; that allocation needs one extra copy of the trainable parameters. Try freeing GPU memory, lowering resolution, or enabling gradient_checkpointing.TÚcandidate_sizer   r#   r$   c                óL   •— t          ‰‰| ‰
‰‰	‰‰‰‰‰‰‰‰‰¬¦  «        }|rdŠ|S )N)ra   r€   rv   r0   r1   r2   rz   r{   F)r¦   )r²   Úokr   rz   r~   r-   r1   r|   r2   r/   r,   r0   r€   rv   ra   r{   s     €€€€€€€€€€€€€€r    Ú_probez%probe_max_micro_batch.<locals>._probe²  sY   ø€ åØØØØØØØØ!1Ø!1Ø+Ø"3Ø&;Ø)Ø-Ø#5ðñ ô ˆBð" ð +Ø%*Ð"ØˆIr   zjauto-batch probe failed at micro_batch_size=1. Try lowering resolution or enabling gradient_checkpointing.z‹auto-batch probe failed at micro_batch_size=1 once optimizer state is resident. Try lowering resolution or enabling gradient_checkpointing.é   r…   )r²   r   r#   r$   )Útyper”   Ú
ValueErrorÚtrainingÚtrainÚ
parametersry   r(   rI   rD   ÚmathÚfloorr’   rA   r‡   rŸ   )r|   r~   r,   r-   r/   r   r0   r1   rª   r«   r2   rz   r¬   r­   rf   r®   Úmodel_trainingÚcriterion_trainingr!   rµ   Úlower_okÚ	candidateÚ
upper_failÚloÚhiÚmidr   r€   rv   ra   r{   s   ````````  ``               @@@@r    Úprobe_max_micro_batchrÆ   K  s  øøøøøøøøøøøøøø€ ð| „{�fÒÐÝÐMÑNÔNÐNØ˜ÒÐÝÐ8Ñ9Ô9Ð9Ø�Ð$Ð$Ò$Ð$ Ò$Ð$Ð$Ð$ÝÐ;Ñ<Ô<Ð<à”^€NØ"Ô+Ðà59ÐØ/3€Mðt!Ø�Š‰ŒˆØ�ŠÑÔÐàMÐM u×'7Ò'7Ñ'9Ô'9ÐMÑMÔMÐØð 	Ýð'ñô ð ð	Ý.EØ  ,Ð0FÐHXÐZiñ/ô /Ñ+Ð˜m˜møõ ð 		ð 		ð 		Ý Ñ$Ô$ð Øõ ðoñô ð ð	øøøøð		øøøð "Ðð	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð@ ˆv�a‰yŒyð 	ÝðNñô ð ð ˆv�a‰yŒyð 	ÝðNñô ð ð
 ˆØˆ	Øˆ
à˜?Ò*Ð*Øˆv�iÑ Ô ð Ø$�Ø˜Q‘�	�	à&�
Øð ˜?Ò*Ð*ð ÐØ(¨1Ñ,ˆJà˜‰\ˆÝ�˜a‘ Ñ1Ô1ˆØ�BŠhˆhØ˜‘7˜q‘.ˆCØˆv�c‰{Œ{ð Ø�Ø˜1‘W��à˜1‘W�ð �BŠhˆhõ ˜q¥$¤*¨X¸Ñ-EÑ"FÔ"FÑGÔGÐØà�Š�NÑ#Ô#Ð#Ø�ŠÐ*Ñ+Ô+Ð+Ø�Š DˆÑ)Ô)Ð)Ø×Ò¨ÐÑ-Ô-Ð-ð Ð'Ø ØÐ$ØÝŒ
×ÒÑ Ô Ð Ð øð 	�Š�NÑ#Ô#Ð#Ø�ŠÐ*Ñ+Ô+Ð+Ø�Š DˆÑ)Ô)Ð)Ø×Ò¨ÐÑ-Ô-Ð-ð Ð'Ø ØÐ$ØÝŒ
×ÒÑ Ô Ð Ð øøøs3   Á/AI- ÃC ÃI- Ã
DÃ( DÄDÄC%I- É-A<K)r   Útarget_effective_batchc                ó    — | dk     rt          d¦  «        ‚|dk     rt          d¦  «        ‚t          dt          j        || z  ¦  «        ¦  «        S )a‚  Recommend gradient accumulation steps to reach target effective batch size.

    Args:
        safe_micro_batch: Per-step batch size that fits in memory.
        target_effective_batch: Desired effective batch (micro_batch * accum_steps).

    Returns:
        ceil(target_effective_batch / safe_micro_batch), at least 1.

    Raises:
        ValueError: If either argument is < 1.
    r)   zsafe_micro_batch must be >= 1.z$target_effective_batch must be >= 1.)r¸   rD   r¼   Úceil)r   rÇ   s     r    Úrecommend_grad_accum_stepsrÊ     sY   € ð ˜!ÒÐÝÐ9Ñ:Ô:Ð:Ø Ò!Ð!ÝÐ?Ñ@Ô@Ð@Ýˆq•$”)Ð2Ð5EÑEÑFÔFÑGÔGÐGr   Úmodel_contextr   Úmodel_configr   Útrain_configr	   c                óª
  — | j         }t          j                             ¦   «         r|j        dk    rt          d¦  «        ‚t          j        t          |dd¦  «        ¦  «        }|t          j	        urct          |dd¦  «        }t          |dd¦  «        }t          |d	d
¦  «        }	t          |j        |||	¦  «        }
|
rt          |
¦  «        n|j        }n|j        }t          |dd¦  «        }t          |dd¦  «        }|�|}t          |dd¦  «        }t          |¦  «        }|rt          |di ¦  «        }nd}t                               d|¦  «         t!          ||¦  «        \  }}|                     |¦  «        }t%          ||d¬¦  «        }|du}|rJ|dk    rt          j        }n9t          j                             ¦   «         rt          j        nt          j        }nd}t-          |¦  «        o+t/          t          |dd¦  «        ¦  «        o|t          j        u }t1          | j        ||||j        ||j        |||t          |dd¦  «        |||¬¦  «        }t          |dd¦  «        }|rRt          |dd¦  «        }t          dt9          j        ||z  ¦  «        ¦  «        }t                               d||¦  «         t          |dd¦  «        }t          |dd¦  «        }t?          |t@          ¦  «        rnt?          |t@          ¦  «        rY|d k    r<t          j                             ¦   «         rt          j         !                    ¦   «         n|}t          d||z  ¦  «        }�n t?          |tD          ¦  «        rét?          |t@          ¦  «        rÔ| #                    ¦   «          $                    ¦   «         }|d!v r?t          j                             ¦   «         rt          j         !                    ¦   «         nd}nW| %                    ¦   «         rtA          |¦  «        }n3d"|v r-tM          d#„ | '                    d"¦  «        D ¦   «         ¦  «        }nd}t          d||z  ¦  «        }nd}|j(        }|dk    r&t          dt9          j)        ||z  ¦  «        ¦  «        } n|} tU          || ¦  «        }!||!z  }"|"|z  }#t          j         +                    |¦  «        }$t                               d$|$||j        ||¦  «         t                               d%||!|"|#¦  «         t                               d&¦  «         t                               d'¦  «         tY          ||!|"|$¬(¦  «        S ))að  Resolve batch_size='auto' into concrete batch_size and grad_accum_steps using a probe.

    Expects model_context to have attributes: .device (torch.device) and .model (nn.Module). Runs probe_max_micro_batch
    on the current model/criterion, then recommend_grad_accum_steps using train_config.auto_batch_target_effective. Logs
    device, segmentation flag, resolution, and the chosen values; also logs that the probe is train-step-only and that
    eval/test may use more memory.

    The optimizer-state memory the probe accounts for (see probe_max_micro_batch) is always modeled as AdamW's, since
    that is the only optimizer the shadow step builds. When train_config.optimizer selects something that is not
    AdamW-shaped (e.g. "sgd", a pytorch-optimizer name, or a custom callable -- all supported by configure_optimizers;
    see _is_adamw_shaped for what still counts as AdamW), this logs a warning: the real optimizer's state memory may
    differ from AdamW's, so the resulting batch size can be too conservative or too permissive. Also re-derives whether
    the real run will use fused AdamW (see
    RFDETRModelModule._use_fused_optimizer) so the shadow optimizer's step-time temporary memory matches it --
    fused and foreach/single-tensor AdamW have different scratch-memory profiles during step().

    Args:
        model_context: Object with .device and .model (e.g. RFDETR.model from get_model()).
        model_config: Architecture config (resolution, num_classes, segmentation_head).
        train_config: Training config (auto_batch_target_effective, amp_dtype); batch_size should be "auto".
        safety_margin: Fraction of max batch to use (passed to probe_max_micro_batch).
        max_micro_batch: Upper bound on batch size to try (passed to probe_max_micro_batch).

    Returns:
        AutoBatchResult with safe_micro_batch, recommended_grad_accum_steps, effective_batch_size, and device_name.

    Raises:
        RuntimeError: If CUDA is not available or model_context.device is not CUDA.
    r‡   z;batch_size='auto' requires a CUDA device for probing in v1.Úmulti_scaleFÚexpanded_scalesTÚ
patch_sizeé   Únum_windowsr8   Ú auto_batch_max_targets_per_imageéd   Úpad_targets_toNrx   r\   rf   a   [auto-batch] optimizer=%r does not allocate AdamW-shaped state; the probe only models AdamW's exp_avg/exp_avg_sq optimizer-state memory (see _build_shadow_optimizer), so this batch-size estimate may be too conservative or too permissive for the optimizer actually configured for training.)Úwarn_legacyÚfp16Úfused_optimizerr2   r*   )r|   r~   r,   r-   r/   r   r0   r1   rª   r«   r2   rz   rf   r®   Úuse_emaÚauto_batch_ema_headroomgffffffæ?r)   z=[auto-batch] Applied EMA headroom (%.2f): safe_micro_batch=%sÚdevicesÚ	num_nodeséÿÿÿÿ)Úautoz-1ú,c                ó:   — g | ]}|                      ¦   «         ¯|‘ŒS r   )r^   )ro   Úentrys     r    rq   z-resolve_auto_batch_config.<locals>.<listcomp>»  s%   € Ð\Ð\Ð\¨%ÈeÏkÊkÉmÌmÐ\ Ð\Ð\Ð\r   za[auto-batch] device=%s world_size=%s segmentation=%s probe_resolution=%s max_targets_per_image=%szl[auto-batch] safe_micro_batch=%s grad_accum_steps=%s effective_batch_per_device=%s global_effective_batch=%szC[auto-batch] This probe estimates train-step-safe micro-batch only.zY[auto-batch] Validation/test (especially segmentation mask eval) may require more memory.)r   r   r   r   )-r-   rA   r‡   Úis_availabler·   r”   r   Ú
from_valueÚgetattrÚOFFr   r,   rD   r`   rœ   r�   r   Útor
   Úfloat16Úis_bf16_supportedÚbfloat16r   r$   rÆ   r|   r/   r0   r¼   r½   Úinfor]   r   Údevice_countr   r^   r&   Úisdigitrž   ÚsplitÚauto_batch_target_effectiverÉ   rÊ   Úget_device_namer   )%rË   rÌ   rÍ   rª   r«   r-   rÏ   rÐ   rÑ   rÓ   ÚscalesÚprobe_resolutionr1   rÖ   rV   Úis_adamw_shapedÚshadow_optimizer_kwargsr~   Ú_Úamp_dtype_strÚamp_enabledÚprobe_autocast_dtypeÚuse_fused_optimizerr   rÚ   ÚheadroomrÜ   rÝ   rì   Ú
world_sizeÚdevices_nameÚtarget_effective_globalÚtarget_effective_per_deviceÚgrad_accum_stepsÚeffective_batch_size_per_deviceÚglobal_effective_batch_sizer   s%                                        r    Úresolve_auto_batch_configr     sŸ  € ðH Ô!€FÝŒ:×"Ò"Ñ$Ô$ð Z¨¬°vÒ(=Ð(=ÝÐXÑYÔYÐYõ Ô'­°¸mÈUÑ(SÔ(SÑTÔT€KØ�*œ.Ð(Ð(Ý! ,Ð0AÀ4ÑHÔHˆÝ˜\¨<¸Ñ<Ô<ˆ
Ý˜l¨M¸1Ñ=Ô=ˆÝ+ØÔ#ØØØñ	
ô 
ˆð +1ÐM�3˜v™;œ;˜;°lÔ6MÐÐà'Ô2Ðå# LÐ2TÐVYÑZÔZÐÝ˜\Ð+;¸TÑBÔB€NØÐ!ð !/Ðå˜L¨+°wÑ?Ô?€Mõ
 ' }Ñ5Ô5€OØð 
õ #*¨,Ð8JÈBÑ"OÔ"OÐÐà"&ÐÝ�Šð0ð ñ	
ô 	
ð 	
õ /¨|¸\ÑJÔJ�L€IˆqØ—’˜VÑ$Ô$€Iå& |°\ÈuÐUÑUÔU€MØ tÐ+€KØð $Ø˜FÒ"Ð"Ý7<´}Ð Ð õ 6;´Z×5QÒ5QÑ5SÔ5SÐ#f¥5¤> >ÕY^ÔYfÐ Ð à#Ðõ 	  Ñ.Ô.ð 	3Ý•˜Ð'8¸$Ñ?Ô?Ñ@Ô@ð	3à ¥E¤NÐ2ð õ -ØÔ!ØØ#ØØ Ô,ØØ&Ô8Ø3Ø#Ø'Ý˜\¨>¸1Ñ=Ô=Ø+Ø0Ø+ðñ ô Ðõ" �l I¨uÑ5Ô5€GØð qÝ˜<Ð)BÀCÑHÔHˆÝ˜q¥$¤*Ð-=ÀÑ-HÑ"IÔ"IÑJÔJÐÝ�ŠÐSÐU]Ð_oÑpÔpÐpõ �l I¨tÑ4Ô4€GÝ˜ k°1Ñ5Ô5€IÝ�'�3ÑÔð ¥J¨y½#Ñ$>Ô$>ð Ø4;¸r²M°MÅeÄj×F]ÒF]ÑF_ÔF_°M•u”z×.Ò.Ñ0Ô0Ð0ÐelˆÝ˜˜L¨9Ñ4Ñ5Ô5ˆ
‰
Ý	�G�SÑ	!Ô	!ð ¥j°½CÑ&@Ô&@ð Ø—}’}‘”×,Ò,Ñ.Ô.ˆØ˜>Ð)Ð)Ý8=¼
×8OÒ8OÑ8QÔ8QÐX�5œ:×2Ò2Ñ4Ô4Ð4ÐWXˆLˆLØ×!Ò!Ñ#Ô#ð 	Ý˜|Ñ,Ô,ˆLˆLØ�LÐ Ð ÝÐ\Ð\°<×3EÒ3EÀcÑ3JÔ3JÐ\Ñ\Ô\Ñ]Ô]ˆLˆLàˆLÝ˜˜L¨9Ñ4Ñ5Ô5ˆ
ˆ
àˆ
ð +ÔFÐØ�A‚~€~Ý&)¨!­T¬YÐ7NÐQ[Ñ7[Ñ-\Ô-\Ñ&]Ô&]Ð#Ð#à&=Ð#å1Ð2BÐD_Ñ`Ô`ÐØ&6Ð9IÑ&IÐ#Ø"AÀJÑ"NÐÝ”*×,Ò,¨VÑ4Ô4€Kå
‡K‚KØkØØØÔ&ØØñô ð õ ‡K‚KØvØØØ'Ø#ñô ð õ ‡K‚KÐUÑVÔVÐVÝ
‡K‚KÐkÑlÔlÐlåØ)Ø%5Ø<Øð	ñ ô ð r   )r!   r"   r#   r$   )Fr)   r*   )r+   r   r,   r   r-   r.   r/   r   r0   r$   r1   r   r2   r   r#   r3   )rV   rW   r#   r$   )NN)ra   rb   rc   rd   re   rd   rf   rg   rh   ri   r#   rj   ) r|   r}   r~   r}   r+   r   r,   r   r-   r.   r/   r   r   r$   ra   rb   r€   r�   rv   r‚   r0   r$   r1   r   r2   r   rz   rƒ   r{   r$   r#   r$   )
Fr)   r§   r¨   r*   Nr©   r©   NN)"r|   r}   r~   r}   r,   r   r-   r.   r/   r   r   r$   r0   r$   r1   r   rª   rd   r«   r   r2   r   rz   rƒ   r¬   rd   r­   rd   rf   rg   r®   ri   r#   r   )r   r   rÇ   r   r#   r   )r§   r¨   )rË   r   rÌ   r   rÍ   r	   rª   rd   r«   r   r#   r   )#r   Ú
__future__r   r¼   Údataclassesr   Útypingr   r   rA   Úrfdetr.configr   r   r	   r
   Úrfdetr.datasets.cocor   Úrfdetr.modelsr   Úrfdetr.training.module_modelr   Úrfdetr.utilities.loggerr   Úrfdetr.utilities.tensorsr   rœ   r   r(   rU   r`   ry   r¦   rÆ   rÊ   r  r   r   r    ú<module>r     sb  ððð ð& #Ð "Ð "Ð "Ð "Ð "à €€€Ø !Ð !Ð !Ð !Ð !Ð !Ø Ð Ð Ð Ð Ð Ð Ð à €€€à RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RØ ;Ð ;Ð ;Ð ;Ð ;Ð ;Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø @Ð @Ð @Ð @Ð @Ð @Ø .Ð .Ð .Ð .Ð .Ð .Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1à	ˆ‰Œ€ð €�$ÐÑÔðð ð ð ð ñ ô ñ Ôðð Pð Pð Pð Pð $Ø!"Øð'ð 'ð 'ð 'ð 'ðT#ð #ð #ð #ðJ /3ØðB$ð B$ð B$ð B$ð B$ðb $Ø!"ØØ)-Ø$ð!wð wð wð wð wð wðB $Ø!"ØØØØ)-ØØ$(Ø.2Ø#'ð!~!ð ~!ð ~!ð ~!ð ~!ðBHð Hð Hð Hð0 ØðEð Eð Eð Eð Eð Eð Er   