# ------------------------------------------------------------------------
# RF-DETR
# Copyright (c) 2025 Roboflow. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 [see LICENSE for details]
# ------------------------------------------------------------------------
"""LightningModule for RF-DETR training and validation."""

from __future__ import annotations

import importlib
import inspect
import math
import random
import warnings
from contextlib import nullcontext
from typing import Any, Callable, cast

import torch
import torch.nn.functional as F  # noqa: N812 -- project-conventional alias (see AGENTS.md)
from pytorch_lightning import LightningModule, seed_everything
from pytorch_lightning.core.optimizer import LightningOptimizer
from pytorch_lightning.utilities.types import LRSchedulerConfigType, OptimizerLRSchedulerConfig
from torch import Tensor
from torch.optim.lr_scheduler import LRScheduler, ReduceLROnPlateau

from rfdetr._namespace import _namespace_from_configs
from rfdetr.config import (
    _MANAGED_SCHEDULER_DEFAULTS,
    ModelConfig,
    MultiScale,
    TrainConfig,
    _is_managed_optimizer_name,
    _is_managed_scheduler_name,
    _resolve_native_optimizer,
)
from rfdetr.datasets.coco import compute_multi_scale_scales
from rfdetr.models.lwdetr import build_criterion_from_config, build_model_from_config
from rfdetr.models.matcher import HungarianMatcher
from rfdetr.models.weights import apply_lora, interpolate_position_embeddings, load_pretrain_weights
from rfdetr.training.callbacks.coco_eval import _get_ema_inner_module
from rfdetr.training.cuda_graph_step import CudaGraphTrainingRunner
from rfdetr.training.param_groups import (
    _build_param_dicts,
    get_param_dict,
    regroup_unmerged_optimizer_state,
    regroup_unmerged_scheduler_kwargs,
)
from rfdetr.utilities.logger import get_logger

logger = get_logger()

_OptimizerFactory = Callable[..., torch.optim.Optimizer]

_TRAIN_PROGRESS_LOSS_ALIASES: dict[str, str] = {
    "loss_ce": "loss_cls",
    "loss_bbox": "loss_box",
    "loss_giou": "loss_giou",
    "loss_mask_ce": "mask_ce",
    "loss_mask_dice": "mask_dice",
    "loss_keypoints_l1": "kp_l1",
    "loss_keypoints_findable": "kp_find",
    "loss_keypoints_visible": "kp_vis",
    "loss_keypoints_nll": "kp_nll",
}


def _is_builtin_fused_adamw(optimizer: object) -> bool:
    """Return whether the config selects RF-DETR's built-in (managed, fused) AdamW path.

    Args:
        optimizer: The ``TrainConfig.optimizer`` value (string or callable).

    Returns:
        ``True`` only for the built-in ``"adamw"`` short name.

    Examples:
        >>> _is_builtin_fused_adamw("adamw")
        True
        >>> _is_builtin_fused_adamw("torch.optim.AdamW")
        False
    """
    return isinstance(optimizer, str) and "." not in optimizer and optimizer.strip().lower() == "adamw"


_FUSED_IGNORED_MSG = (
    "fused_optimizer=True is ignored for optimizer=%r; the fused AdamW kernel only applies to the "
    "built-in optimizer='adamw' path."
)


def _import_optimizer_class(dotted_path: str) -> _OptimizerFactory:
    """Import an optimizer class or factory from a dotted path.

    Args:
        dotted_path: Fully-qualified path such as ``"torch.optim.AdamW"`` or
            ``"pytorch_optimizer.Lion"``.

    Returns:
        The imported optimizer class or factory.

    Raises:
        ValueError: If the module or attribute cannot be imported.
    """
    module_path, _, attribute = dotted_path.rpartition(".")
    if not module_path:
        raise ValueError(f"optimizer {dotted_path!r} is not a valid dotted import path.")
    try:
        module = importlib.import_module(module_path)
        return cast(_OptimizerFactory, getattr(module, attribute))
    except (ImportError, AttributeError) as exc:
        raise ValueError(
            f"Could not import optimizer {dotted_path!r}: {exc}. "
            "Use a fully-qualified path to an importable optimizer class, e.g. 'torch.optim.AdamW'."
        ) from exc


def _instantiate_explicit_optimizer(
    optimizer_class: _OptimizerFactory,
    optimizer_name: str,
    param_dicts: list[dict[str, Any]],
    optimizer_kwargs: dict[str, Any],
) -> torch.optim.Optimizer:
    """Instantiate an explicitly-selected optimizer from param groups and kwargs only.

    Explicit optimizers (dotted import paths and callables) receive the RF-DETR
    parameter groups — which already carry per-group learning rates — plus the
    user's ``optimizer_kwargs`` verbatim. RF-DETR injects no ``lr`` or
    ``weight_decay`` of its own.

    Args:
        optimizer_class: Optimizer class or factory to instantiate.
        optimizer_name: Name used in error messages.
        param_dicts: RF-DETR parameter groups with layer-wise learning rates.
        optimizer_kwargs: Keyword arguments forwarded verbatim to the constructor.

    Returns:
        Instantiated optimizer.

    Raises:
        TypeError | ValueError: Re-raised with an RF-DETR-specific hint on failure.
    """
    try:
        return optimizer_class(param_dicts, **optimizer_kwargs)
    except (TypeError, ValueError) as exc:
        raise type(exc)(
            f"Failed to initialize optimizer {optimizer_name!r}: {exc}. "
            "Explicit optimizers (dotted paths and callables) are built from the RF-DETR parameter "
            "groups plus your `optimizer_kwargs` only, with no lr/weight_decay injected. Check that the "
            "class accepts these arguments, or pass a callable/functools.partial needing only `params`."
        ) from exc


def _optimizer_accepts_kwarg(optimizer_class: _OptimizerFactory, name: str) -> bool:
    """Return whether an optimizer constructor accepts a given keyword argument.

    Args:
        optimizer_class: Optimizer class or factory to inspect.
        name: Keyword-argument name to look for.

    Returns:
        ``True`` when the constructor declares ``name`` or accepts arbitrary
        keyword arguments, or when its signature cannot be introspected (e.g. a
        C-implemented constructor) — in which case the constructor validates the
        call itself.

    Examples:
        >>> _optimizer_accepts_kwarg(torch.optim.SGD, "weight_decay")
        True
    """
    try:
        signature = inspect.signature(optimizer_class)
    except (TypeError, ValueError):
        return True
    parameters = signature.parameters.values()
    if any(parameter.kind is inspect.Parameter.VAR_KEYWORD for parameter in parameters):
        return True
    return name in signature.parameters


def _instantiate_optimizer(
    optimizer_class: _OptimizerFactory,
    optimizer_name: str,
    param_dicts: list[dict[str, Any]],
    train_config: TrainConfig,
) -> torch.optim.Optimizer:
    """Instantiate an optimizer class with RF-DETR optimizer arguments.

    ``weight_decay`` is injected only when the optimizer constructor accepts it,
    so optimizers with a different regularization API are not forced to fail.

    Args:
        optimizer_class: Optimizer class or factory to instantiate.
        optimizer_name: Name used in error messages.
        param_dicts: RF-DETR parameter groups with layer-wise learning rates.
        train_config: Training config with base optimizer hyperparameters.

    Returns:
        Instantiated optimizer.

    Raises:
        TypeError | ValueError: Re-raised with an RF-DETR-specific hint when the
            optimizer constructor rejects the supplied arguments.
    """
    init_kwargs: dict[str, Any] = {"lr": train_config.lr}
    if _optimizer_accepts_kwarg(optimizer_class, "weight_decay"):
        init_kwargs["weight_decay"] = train_config.weight_decay
    init_kwargs.update(train_config.optimizer_kwargs)
    try:
        return optimizer_class(param_dicts, **init_kwargs)
    except (TypeError, ValueError) as exc:
        raise type(exc)(
            f"Failed to initialize optimizer {optimizer_name!r}: {exc}. "
            "For managed torch.optim short names, RF-DETR passes `params`, `lr`, and (when supported) "
            "`weight_decay`, then your `optimizer_kwargs`; this usually means an unsupported entry in "
            "`optimizer_kwargs`."
        ) from exc


_SchedulerFactory = Callable[..., LRScheduler | ReduceLROnPlateau]


def _import_scheduler_class(dotted_path: str) -> _SchedulerFactory:
    """Import an LR-scheduler class or factory from a dotted path.

    Args:
        dotted_path: Fully-qualified path such as ``"torch.optim.lr_scheduler.StepLR"`` or
            ``"pytorch_optimizer.CosineAnnealingWarmupRestarts"``.

    Returns:
        The imported scheduler class or factory.

    Raises:
        ValueError: If the module or attribute cannot be imported.
    """
    module_path, _, attribute = dotted_path.rpartition(".")
    if not module_path:
        raise ValueError(f"lr_scheduler {dotted_path!r} is not a valid dotted import path.")
    try:
        module = importlib.import_module(module_path)
        return cast(_SchedulerFactory, getattr(module, attribute))
    except (ImportError, AttributeError) as exc:
        raise ValueError(
            f"Could not import lr_scheduler {dotted_path!r}: {exc}. "
            "Use a fully-qualified path to an importable scheduler class, e.g. 'torch.optim.lr_scheduler.StepLR'."
        ) from exc


def _instantiate_explicit_scheduler(
    scheduler_factory: _SchedulerFactory,
    scheduler_name: str,
    optimizer: torch.optim.Optimizer,
    scheduler_kwargs: dict[str, Any],
) -> LRScheduler | ReduceLROnPlateau:
    """Instantiate an explicitly-selected LR scheduler from the optimizer and kwargs only.

    Explicit schedulers (dotted import paths and callables) receive the built optimizer plus the
    user's ``lr_scheduler_kwargs`` verbatim. RF-DETR injects no ``total_steps`` / ``T_max`` of its
    own — the managed ``"step"`` / ``"cosine"`` presets remain the runtime-aware option.

    Args:
        scheduler_factory: Scheduler class or factory to instantiate.
        scheduler_name: Name used in error messages.
        optimizer: The optimizer the scheduler drives.
        scheduler_kwargs: Keyword arguments forwarded verbatim to the constructor.

    Returns:
        Instantiated scheduler.

    Raises:
        TypeError | ValueError: Re-raised with an RF-DETR-specific hint on failure.
    """
    try:
        return scheduler_factory(optimizer, **scheduler_kwargs)
    except (TypeError, ValueError) as exc:
        raise type(exc)(
            f"Failed to initialize lr_scheduler {scheduler_name!r}: {exc}. "
            "Explicit schedulers (dotted paths and callables) are built from the optimizer plus your "
            "`lr_scheduler_kwargs` only, with no total_steps/T_max injected. Check that the class accepts these "
            "arguments, or pass a callable/functools.partial needing only the optimizer."
        ) from exc


def _build_managed_scheduler(
    optimizer: torch.optim.Optimizer,
    train_config: TrainConfig,
    total_steps: int,
    steps_per_epoch: int,
    warmup_steps: int,
) -> torch.optim.lr_scheduler.LambdaLR:
    """Build the managed ``"step"`` / ``"cosine"`` scheduler as a warmup-aware ``LambdaLR``.

    Preserves RF-DETR's built-in schedule: a linear warmup ramp over ``warmup_steps`` followed by
    either cosine annealing down to ``min_factor`` or a 10x step decay after ``lr_drop`` epochs. The
    ``min_factor`` and ``lr_drop`` values are read from ``lr_scheduler_kwargs``, falling back to
    ``_MANAGED_SCHEDULER_DEFAULTS`` when a key is absent.

    Args:
        optimizer: The optimizer the scheduler drives.
        train_config: Training config carrying the preset name and schedule knobs.
        total_steps: Total optimizer steps over the whole run.
        steps_per_epoch: Optimizer steps per epoch.
        warmup_steps: Number of optimizer steps in the linear warmup ramp.

    Returns:
        A ``LambdaLR`` implementing the managed schedule.
    """
    kwargs = train_config.lr_scheduler_kwargs
    # Managed presets are always strings (guaranteed by the _is_managed_scheduler_name branch at the call site).
    preset = cast(str, train_config.lr_scheduler).strip().lower()
    min_factor = float(kwargs.get("min_factor", _MANAGED_SCHEDULER_DEFAULTS["min_factor"]))
    lr_drop = int(kwargs.get("lr_drop", _MANAGED_SCHEDULER_DEFAULTS["lr_drop"]))

    def lr_lambda(current_step: int) -> float:
        if current_step < warmup_steps:
            return float(current_step) / float(max(1, warmup_steps))
        if preset == "cosine":
            progress = float(current_step - warmup_steps) / float(max(1, total_steps - warmup_steps))
            return min_factor + (1 - min_factor) * 0.5 * (1 + math.cos(math.pi * progress))
        # Step decay: drop by 10x after lr_drop epochs.
        if current_step < lr_drop * steps_per_epoch:
            return 1.0
        return 0.1

    return torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lr_lambda)


def _wrap_with_warmup(
    scheduler: torch.optim.lr_scheduler.LRScheduler,
    optimizer: torch.optim.Optimizer,
    warmup_steps: int,
) -> torch.optim.lr_scheduler.SequentialLR:
    """Prepend a linear warmup ramp to an explicit scheduler via ``SequentialLR``.

    The warmup ramps the LR from ``1 / warmup_steps`` of its base value up to full over
    ``warmup_steps`` optimizer steps, then hands control to ``scheduler``. ``ReduceLROnPlateau`` is
    metric-driven and cannot be composed this way — callers must skip wrapping it.

    Args:
        scheduler: The explicit scheduler to run after warmup.
        optimizer: The optimizer both schedulers drive.
        warmup_steps: Number of optimizer steps in the linear warmup ramp.

    Returns:
        A ``SequentialLR`` chaining the warmup ramp and ``scheduler``.
    """
    warmup = torch.optim.lr_scheduler.LinearLR(
        optimizer,
        start_factor=1.0 / max(1, warmup_steps),
        end_factor=1.0,
        total_iters=warmup_steps,
    )
    return torch.optim.lr_scheduler.SequentialLR(
        optimizer,
        schedulers=[warmup, scheduler],
        milestones=[warmup_steps],
    )


class RFDETRModelModule(LightningModule):
    """LightningModule wrapping the RF-DETR model and training loop.

    Args:
        model_config: Architecture configuration.
        train_config: Training hyperparameter configuration.
    """

    def __init__(self, model_config: ModelConfig, train_config: TrainConfig) -> None:
        super().__init__()
        self.model_config = model_config
        self.train_config = train_config
        # Manual optimization is enabled only for keypoint models so that the box-count
        # normalizer can be accumulated across grad-accum microbatches. Detection and
        # segmentation use Lightning's automatic optimization (PTL handles accumulation,
        # AMP, and gradient clipping), which keeps their step semantics unchanged from
        # the pre-fix/scaling behaviour.
        self._use_manual_optimization: bool = bool(getattr(model_config, "use_grouppose_keypoints", False))
        self.automatic_optimization = not self._use_manual_optimization
        # LR-scheduler stepping cadence resolved in configure_optimizers(); read by the manual-optimization
        # step loop and the epoch-end hook. Defaults keep pre-configure behaviour (per-step stepping).
        self._lr_scheduler_interval: str = "step"
        self._lr_scheduler_monitor: str | None = None
        self._accumulated_box_normalizer: Tensor | None = None
        # One-shot guard for the notice announcing that the "auto" validation-loss policy resolved to skipping the
        # loss; emitted from on_validation_epoch_start on the first real (non-sanity) validation epoch only.
        self._logged_val_loss_skip_notice: bool = False
        # Validation-loss policy resolved once per validation epoch by on_validation_epoch_start and read per batch by
        # _should_compute_val_loss. None until that hook runs, so direct validation_step calls resolve it themselves.
        self._resolved_compute_val_loss: bool | None = None
        # Memoized loss_name -> aggregate train/ key (or None for a standalone key), built lazily by
        # _aux_aggregate_map() and recomputed only when loss_dict's key set changes between calls.
        self._aux_aggregate_cache: dict[str, str | None] | None = None
        self._aux_aggregate_cache_keys: frozenset[str] | None = None
        self._cuda_graph_runner: CudaGraphTrainingRunner | None = None
        # True when cuda_graphs and compile are both active: Inductor's CUDA graph trees replay the
        # compiled kernels, so _configure_cuda_graph_runner leaves the eager runner unset and
        # training_step marks each step for the graph-tree allocator instead.
        self._inductor_cudagraphs: bool = False
        # Compilation can remain enabled after the narrower Inductor graph-tree gate declines a request.
        # Keep this separate so the eager runner never captures an OptimizedModule in a compile-only fallback.
        self._compile_active: bool = False
        # Allow partial state-dict loading when resuming from a .pth checkpoint
        # (which contains only model weights, not criterion/postprocess state).
        self.strict_loading = False

        # Model, criterion, and postprocessor.
        self.model = build_model_from_config(model_config, train_config)
        if model_config.pretrain_weights is not None:
            # Canonical loader handles PE interpolation, PTL .ckpt normalisation,
            # per-group query slicing, class-name extraction, partial-load warnings,
            # and writes any auto-aligned ``num_classes`` back onto ``model_config``.
            load_pretrain_weights(self.model, self.model_config)
            if model_config.use_grouppose_keypoints:
                # Older model shims may omit the keypoint reset hook; call it only when implemented.
                reset_keypoint_gaussian_parameters = getattr(self.model, "reset_keypoint_gaussian_parameters", None)
                if callable(reset_keypoint_gaussian_parameters):
                    reset_keypoint_gaussian_parameters()
                    logger.info(
                        "Reset keypoint Gaussian precision outputs to unit values after pretrained weight load."
                    )
        if model_config.backbone_lora:
            apply_lora(self.model)

        # Build criterion/postprocessors after potential num_classes alignment so
        # they are constructed with a config that matches the current model head.
        self.criterion, self.postprocess = build_criterion_from_config(self.model_config, self.train_config)

        # torch.compile is opt-in: set model_config.compile=True to enable.
        # Only enabled on CUDA; MPS and CPU do not benefit from compilation.
        # Use the fork-safe DEVICE constant instead of torch.cuda.is_available(),
        # which creates a CUDA driver context that breaks fork-based DDP.
        from rfdetr.config import DEVICE

        accelerator = str(train_config.accelerator).lower()
        uses_cuda_accelerator = accelerator in {"auto", "gpu", "cuda"}
        # multi_scale is deliberately not part of this gate. The XLA concern it used to carry
        # (one graph trace per scale) cannot arise here: DEVICE == "cuda" and an accelerator of
        # auto/gpu/cuda are both required first, so an XLA or TPU run has already disabled
        # compilation before multi-scale is even considered. Excluding it only ever disabled the
        # CUDA path, where dynamic=True below is precisely what handles the varying (H, W).
        compile_enabled = model_config.compile and DEVICE == "cuda" and uses_cuda_accelerator
        if model_config.compile and not compile_enabled:
            logger.info(
                "Disabling torch.compile: RF-DETR enables it only on a CUDA device with a "
                "CUDA-family accelerator (got DEVICE=%r, accelerator=%r). CPU and MPS are not "
                "expected to benefit; on XLA/TPU the graph is compiled by the XLA runtime instead.",
                DEVICE,
                accelerator,
            )
        if compile_enabled:
            # Dynamic shapes let one graph handle all multi-scale input sizes instead of
            # recompiling per (H, W) pair. Fixed-resolution training uses a static graph so
            # Inductor can specialize dimensions that stay fixed across training batches
            # (a validation batch of another shape recompiles once). Positional
            # interpolation has its own eager boundary for unsupported symbolic bicubic backward.
            # Do not suppress other compiler errors: nested retries can flood logs and conceal
            # lost acceleration.
            # capture_scalar_outputs=True: include Tensor.item() calls. The per-level spatial
            # shapes reach gen_encoder_output_proposals and ms_deform_attn as Python ints
            # (spatial_shapes_hw), which are symbols under dynamic tracing and constants in a
            # static specialization.
            torch._dynamo.config.capture_scalar_outputs = True
            # Inductor's coalesce tiling analysis is unsupported on the dynamic-shape path
            # (torch/_inductor/config.py: "coalesce_tiling_analysis does not yet apply to
            # dynamic shapes"), yet it still runs and reaches an assert in
            # tiling_utils.get_pw_red_splits comparing size hints. That assert has no
            # symbolic-shape escape, unlike the CantSplit branch below it, so entire forward
            # frames fall back to eager. Turning the analysis off costs nothing under
            # dynamic=True. Passed as a compile option rather than assigned on the inductor
            # config module, so the default is preserved for any other compilation in this
            # process. The knob is absent on older torch versions, where passing it would raise
            # RuntimeError("Unexpected optimization option ..."), hence the hasattr guard.
            # Local import: pulls in inductor, which an uncompiled run never needs.
            import torch._inductor.config as inductor_config

            compile_options: dict[str, Any] = {}
            if hasattr(inductor_config.triton, "coalesce_tiling_analysis"):
                compile_options["triton.coalesce_tiling_analysis"] = False
            if model_config.cuda_graphs:
                # Replay the compiled kernels with Inductor's CUDA graph trees. This is what
                # mode="reduce-overhead" sets, but torch.compile rejects mode= together with
                # options=, so the option key is passed directly. The eager CudaGraphTrainingRunner
                # must not wrap the OptimizedModule on top of this: make_graphed_callables would
                # capture Inductor's launch path a second time (see _configure_cuda_graph_runner).
                unsupported_reason = self._inductor_cudagraphs_unsupported_reason(model_config, train_config)
                if unsupported_reason is None:
                    compile_options["triton.cudagraphs"] = True
                    self._inductor_cudagraphs = True
                    logger.info(
                        "CUDA graph replay of the compiled model enabled through Inductor cudagraph trees; "
                        "each new input shape records its own graph (TORCH_LOGS=cudagraphs shows partitions)."
                    )
                else:
                    logger.warning(
                        "Falling back to compile-only because %s; cuda_graphs together with compile is "
                        "validated for single-GPU detection training without gradient accumulation.",
                        unsupported_reason,
                    )
            # Duck-typed like the criterion capability probes below: test doubles and custom criteria
            # need not expose a matcher at all.
            matcher = getattr(self.criterion, "matcher", None)
            if isinstance(matcher, HungarianMatcher):
                # The matcher owns its compile recipe and builds the compiled cost lazily in the process that
                # runs it, so spawn-based strategies can still pickle the criterion into their workers. It
                # shares the model's Inductor options (the coalesce-tiling workaround applies to its dynamic
                # graph too) and forces CUDA graph replay off on top of them itself.
                matcher.enable_compiled_l1_cost(compile_options)
                logger.info("Matcher L1 compilation enabled (dynamic shapes, no CUDA graphs; kernels compile on use).")
            # OptimizedModule forwards attribute access to the wrapped LWDETR via
            # __getattr__ at runtime, so self.model keeps working everywhere it's used below.
            # Static specialization is measured only for the ordinary fixed-resolution detection compile path.
            # Aspect-preserving resize (square_resize_div_64=False) pads each batch to its own (H, W),
            # Inductor CUDA graphs keep their established recipe, and segmentation and keypoint models
            # were not measured, so all of them stay on the dynamic compile.
            dynamic_shapes = (
                train_config.multi_scale is not MultiScale.OFF
                or not train_config.square_resize_div_64
                or bool(train_config.aug_config)
                or model_config.cuda_graphs
                or model_config.segmentation_head
                or model_config.use_grouppose_keypoints
            )
            enable_compiled_losses = getattr(self.criterion, "enable_compiled_detection_losses", None)
            if enable_compiled_losses is not None and not dynamic_shapes:
                enable_compiled_losses()
            self.model = torch.compile(  # type: ignore[assignment]
                self.model, dynamic=dynamic_shapes, options=compile_options or None
            )
            self._compile_active = True

    @staticmethod
    def _inductor_cudagraphs_unsupported_reason(model_config: ModelConfig, train_config: TrainConfig) -> str | None:
        """Explain why Inductor CUDA graph replay must stay off for this run, or return ``None`` when it may run.

        Mirrors the single-GPU detection scope of :class:`CudaGraphTrainingRunner` from what is known at construction
        time, plus one limit specific to cudagraph trees: the compiled backward allocates its gradient outputs inside
        the graph pool, so a ``.grad`` adopted by the first microbatch is overwritten by the next replay and gradient
        accumulation raises on the second microbatch.

        Args:
            model_config: Model configuration being built.
            train_config: Training configuration for this fit run.

        Returns:
            A human-readable reason, or ``None`` when the combined path is within its validated scope.
        """
        if not hasattr(torch.compiler, "cudagraph_mark_step_begin"):
            return "this torch version has no torch.compiler.cudagraph_mark_step_begin"
        if train_config.amp_dtype == "fp8":
            return "FP8 graph replay requires Transformer Engine capture with compile=False"
        if model_config.segmentation_head:
            return "segmentation training is not supported"
        if model_config.use_grouppose_keypoints:
            return "keypoint training is not supported"
        if model_config.gradient_checkpointing:
            return "gradient checkpointing is not supported"
        if int(train_config.grad_accum_steps) > 1:
            return f"gradient accumulation (grad_accum_steps={train_config.grad_accum_steps}) is not supported"
        devices = train_config.devices
        multi_device = int(train_config.num_nodes) > 1 or (isinstance(devices, int) and devices > 1)
        if isinstance(devices, str) and devices not in {"1", "auto"}:
            multi_device = True
        if multi_device:
            return f"distributed training (devices={devices!r}, num_nodes={train_config.num_nodes}) is not supported"
        return None

    # ------------------------------------------------------------------
    # PTL lifecycle hooks
    # ------------------------------------------------------------------

    def on_fit_start(self) -> None:
        """Validate loss consumers and seed RNGs at fit start.

        Rejects ``compute_val_loss=False`` when a configured callback or scheduler
        consumes ``val/loss``. Seeding here avoids hidden global side-effects in
        ``build_trainer`` while preserving deterministic training behaviour for fit runs.

        Raises:
            ValueError: If a callback or scheduler monitors ``val/loss`` while validation-loss computation is disabled.
        """
        if self.train_config.compute_val_loss is False and self._validation_loss_is_monitored:
            raise ValueError(
                "compute_val_loss=False is incompatible with a callback or scheduler monitoring 'val/loss'. "
                "Set compute_val_loss=True or 'auto', or monitor a metric that is produced."
            )
        if self.train_config.seed is not None:
            seed_everything(self.train_config.seed + self.global_rank, workers=True)

    def on_train_start(self) -> None:
        """Configure the CUDA graph runner, then normalize restored fused-optimizer state.

        Device and world-size placement are only final once Lightning reaches this hook, so
        :meth:`_configure_cuda_graph_runner` is called first to decide, for this fit run, whether ``training_step``
        replays a captured graph or stays eager.

        Lightning restores optimizer state after ``on_fit_start``.  Fused AdamW is strict about the dtype, device, and
        layout of its moment tensors, so resuming from a checkpoint can fail if Lightning rehydrates those tensors in a
        layout that no longer matches the live parameters.  Recasting same-shaped floating-point tensors here keeps the
        resumed optimizer compatible without discarding the saved momentum state.
        """
        self._configure_cuda_graph_runner()

        if not self._use_fused_optimizer:
            return

        try:
            optimizers = self.optimizers(use_pl_optimizer=False)
        except RuntimeError:
            return
        if optimizers is None:
            return
        if isinstance(optimizers, list):
            optimizer_list = optimizers
        else:
            optimizer_list = [optimizers]

        normalized_tensors = 0
        for optimizer in optimizer_list:
            if not isinstance(optimizer, torch.optim.Optimizer):
                optimizer = getattr(optimizer, "optimizer", optimizer)
            if isinstance(optimizer, torch.optim.Optimizer):
                normalized_tensors += self._normalize_optimizer_state(optimizer)
        if normalized_tensors and getattr(self.trainer, "is_global_zero", True):
            logger.info(
                "Normalized %d restored fused AdamW state tensors after checkpoint resume.",
                normalized_tensors,
            )

    def _configure_cuda_graph_runner(self) -> None:
        """Enable CUDA graph replay only for its validated single-GPU detection scope.

        Raises:
            RuntimeError: If Inductor CUDA graph replay was compiled in but the trainer resolved gradient
                accumulation, more than one process, or Transformer Engine precision; or if the FP8 capture
                plugin has no recipe or a compatible Transformer Engine capture API is unavailable.
        """
        self._cuda_graph_runner = None
        if not getattr(self.model_config, "cuda_graphs", False):
            return
        if self._compile_active and not self._inductor_cudagraphs:
            logger.info(
                "CUDA graphs requested with compilation outside Inductor's validated scope; "
                "training remains compile-only."
            )
            return
        if self._inductor_cudagraphs:
            # The construction-time gate read TrainConfig; trainer_kwargs can override
            # accumulate_grad_batches and devices="auto" can resolve to several GPUs. The model is
            # already compiled with cudagraphs, so the only safe response now is to stop.
            if self.device.type != "cuda":
                self._inductor_cudagraphs = False
                logger.warning(
                    "Disabling Inductor CUDA graph replay because the model is on %r, not CUDA; "
                    "training will remain compile-only.",
                    self.device.type,
                )
                return
            accumulate_grad_batches = int(getattr(self.trainer, "accumulate_grad_batches", 1))
            world_size = int(getattr(self.trainer, "world_size", 1))
            if accumulate_grad_batches > 1 or world_size != 1:
                raise RuntimeError(
                    "cuda_graphs=True with compile=True is validated for single-GPU training without gradient "
                    f"accumulation, but the trainer resolved accumulate_grad_batches={accumulate_grad_batches} and "
                    f"world_size={world_size}. Set cuda_graphs=False or drop the accumulation / extra devices."
                )
            if str(getattr(self.trainer, "precision", "")).startswith("transformer-engine"):
                raise RuntimeError(
                    "FP8 CUDA graphs require compile=False and Transformer Engine capture; "
                    "the trainer precision cannot override an already compiled Inductor graph run."
                )
            logger.info(
                "CUDA graph replay is handled by Inductor cudagraph trees for the compiled model on %s; "
                "the eager graph runner stays off.",
                self.device,
            )
            return

        unsupported_reason: str | None = None
        if self.device.type != "cuda":
            unsupported_reason = f"the model is on {self.device.type!r}, not CUDA"
        elif int(getattr(self.trainer, "world_size", 1)) != 1:
            unsupported_reason = "distributed training is not supported"
        elif self.model_config.segmentation_head:
            unsupported_reason = "segmentation training is not supported"
        elif self.model_config.use_grouppose_keypoints:
            unsupported_reason = "keypoint training is not supported"
        elif self.model_config.gradient_checkpointing:
            unsupported_reason = "gradient checkpointing is not supported"
        elif str(self.trainer.precision) == "transformer-engine":
            if not self.train_config.square_resize_div_64:
                unsupported_reason = "FP8 capture requires square_resize_div_64=True"
            elif self.train_config.multi_scale is not MultiScale.OFF:
                unsupported_reason = "FP8 capture requires multi_scale=False"
            elif int(getattr(self.trainer, "accumulate_grad_batches", self.train_config.grad_accum_steps)) != 1:
                unsupported_reason = "FP8 capture does not support gradient accumulation"
        elif str(self.trainer.precision) not in {"bf16-mixed", "bf16-true"}:
            unsupported_reason = (
                f"the trainer precision is {self.trainer.precision!r}; capture requires BF16 or Transformer Engine FP8"
            )

        if unsupported_reason is not None:
            logger.warning("Disabling CUDA graphs because %s; training will run eagerly.", unsupported_reason)
            return
        if str(self.trainer.precision) == "transformer-engine":
            recipe = getattr(self.trainer.precision_plugin, "recipe", None)
            if recipe is None:
                raise RuntimeError("FP8 CUDA graph capture requires the active Lightning Transformer Engine recipe.")
            self._cuda_graph_runner = CudaGraphTrainingRunner(self.model, fp8_recipe=recipe)
        else:
            self._cuda_graph_runner = CudaGraphTrainingRunner(self.model)
        logger.info(
            "CUDA graph replay enabled for the training forward on %s (%s); the first batch of each input "
            "shape runs eager warm-up and capture.",
            self.device,
            self.trainer.precision,
        )

    def on_train_batch_start(self, batch: tuple[Any, Any], batch_idx: int) -> None:
        """Apply optional multi-scale resize to the incoming batch.

        Modifications to ``batch`` (in-place on ``NestedTensor``) are visible in ``training_step`` because they share
        the same object.

        Args:
            batch: Tuple of (NestedTensor samples, list of target dicts).
            batch_idx: Index of the current batch within the epoch.
        """
        tc = self.train_config
        mc = self.model_config

        if tc.multi_scale is MultiScale.PER_BATCH:
            samples, _ = batch
            scales = compute_multi_scale_scales(mc.resolution, tc.expanded_scales, mc.patch_size, mc.num_windows)
            step = self.trainer.global_step
            # Use a step-local generator so the scale choice is deterministic and DDP-consistent
            # without reseeding the process-global RNG on every batch.
            scale = random.Random(step).choice(scales)
            with torch.no_grad():
                samples.tensors = F.interpolate(samples.tensors, size=scale, mode="bilinear", align_corners=False)
                samples.mask = (
                    F.interpolate(samples.mask.unsqueeze(1).float(), size=scale, mode="nearest").squeeze(1).bool()
                )

    def on_train_epoch_start(self) -> None:
        """Reset the accumulated box normalizer at the start of every training epoch.

        Lightning may reuse the module across epochs without calling ``_step_optimizer`` at the boundary (for example
        when an epoch ends mid-accumulation window with a non-divisible batch count). Clearing the accumulator here
        guarantees the manual-optimization path always starts each epoch from a known state, so the first microbatch's
        gradients are scaled by its own box count and not by a stale previous-epoch denominator.

        This is a no-op for non-keypoint models because they use Lightning's automatic optimization path and never
        populate ``self._accumulated_box_normalizer``.

        Note: on finite datasets the final-batch fallback in ``_should_step_optimizer`` always flushes a partial
        trailing window, so this reset is the only change needed.  On IterableDatasets (infinite
        ``num_training_batches``) a partial window may survive epoch end with un-stepped gradients; those are
        discarded here and the optimizer is zeroed so the first microbatch of the new epoch starts from a clean state.
        """
        if self._accumulated_box_normalizer is not None:
            # Discard any partial accumulation window that survived the epoch boundary
            # (only possible for IterableDatasets where num_training_batches is infinite).
            try:
                opts = self.optimizers()
                for opt in opts if isinstance(opts, list) else [opts]:
                    opt.zero_grad()
            except RuntimeError:
                pass  # Not attached to Trainer (unit-test context); nothing to zero.
        self._accumulated_box_normalizer = None

    def training_step(self, batch: tuple[Any, Any], batch_idx: int) -> Tensor | dict[str, Any]:
        """Compute loss for one training step and log metrics.

        PTL handles AMP (``precision``) without a manual ``GradScaler``. Keypoint models perform manual optimization so
        box-count loss normalization is based on the full accumulated effective batch rather than each microbatch
        independently; detection and segmentation models keep Lightning's automatic optimization path.

        Args:
            batch: Tuple of (NestedTensor samples, list of target dicts).
            batch_idx: Batch index within the epoch.

        Returns:
            Scalar loss tensor by default. When ``compute_train_metrics=True``,
            returns a Lightning-compatible dict containing ``loss`` plus
            detached postprocessed predictions for train mAP logging.
        """
        samples, targets = batch
        batch_size = len(targets)
        if self._inductor_cudagraphs:
            # Lightning holds the logged loss tensors past this step. Marking the step lets cudagraph
            # trees reuse the previous step's output memory instead of raising "accessing tensor
            # output of CUDAGraphs that has been overwritten by a subsequent run" on the next replay.
            torch.compiler.cudagraph_mark_step_begin()  # type: ignore[no-untyped-call]
        outputs = (
            self._cuda_graph_runner(samples, targets)
            if self._cuda_graph_runner is not None
            else self.model(samples, targets)
        )
        if self._use_manual_optimization:
            loss_dict, raw_loss, normalizer = self._compute_train_losses(outputs, targets)
            loss_for_backward = self._scale_loss_for_accumulation(raw_loss, normalizer)
        else:
            loss_dict = self.criterion(outputs, targets)
            loss_for_backward = None
        weight_dict = self.criterion.weight_dict
        loss: Tensor = torch.stack([loss_dict[k] * weight_dict[k] for k in loss_dict if k in weight_dict]).sum()
        # Automatic optimization path: return the loss unscaled. Lightning divides the returned loss by
        # ``trainer.accumulate_grad_batches`` itself (``ClosureResult.from_training_step_output``) before
        # ``backward()``, so the accumulated gradient already equals the mean over the window; dividing here as
        # well scaled every accumulated gradient by ``1/N**2``. The manual path scales its own backward loss above.
        train_log_sync_dist = bool(self.train_config.train_log_sync_dist)
        train_log_on_step = bool(self.train_config.train_log_on_step)
        if self.train_config.compact_train_metrics:
            train_loss_metrics = self._compact_train_loss_metrics(loss_dict, weight_dict)
            component_on_step = False
        else:
            train_loss_metrics = {f"train/{loss_name}": value for loss_name, value in loss_dict.items()}
            component_on_step = train_log_on_step
        self.log_dict(
            train_loss_metrics,
            on_step=component_on_step,
            on_epoch=True,
            sync_dist=train_log_sync_dist,
            batch_size=batch_size,
        )
        self.log(
            "train/loss",
            loss,
            prog_bar=True,
            on_step=train_log_on_step,
            on_epoch=True,
            sync_dist=train_log_sync_dist,
            batch_size=batch_size,
        )
        self._log_train_progress_metrics(loss, loss_dict, batch_size=batch_size)
        if self._use_manual_optimization:
            # Only the manual path drives the optimizer itself; Lightning owns it on the
            # automatic path, so fetching it there would touch trainer.strategy for nothing.
            optimizer = self.optimizers()
            if isinstance(optimizer, list):
                optimizer = optimizer[0]
            # loss_for_backward is only None in the automatic-optimization branch above,
            # which is mutually exclusive with _use_manual_optimization.
            assert loss_for_backward is not None
            should_step = self._should_step_optimizer(batch_idx)
            # LightningOptimizer maps sync_grad=False to DDP's no_sync context. Intermediate
            # microbatches accumulate locally; the closing backward reduces the whole window.
            sync_context = (
                optimizer.toggle_model(sync_grad=should_step)
                if isinstance(optimizer, LightningOptimizer)
                else nullcontext()
            )
            with sync_context:
                self.manual_backward(loss_for_backward)
            if should_step:
                self._step_optimizer(optimizer)
        if self.train_config.compute_train_metrics:
            with torch.no_grad():
                orig_sizes = torch.stack([t["orig_size"] for t in targets])
                # Slice to group-0 queries only — mirrors the eval-mode path in
                # lwdetr.py that trims refpoint_embed to [:num_queries]. Without
                # this, training mode emits group_detr×num_queries queries (e.g.
                # 13×300=3900) and postprocess top-k selection draws from all
                # groups, producing OKS/mAP values ~50× below true accuracy.
                nq = self.model_config.num_queries
                # Only include tensor-valued keys — pred_masks is a dict in
                # train mode (sparse_forward) and postprocess cannot handle it.
                inference_outputs = {
                    k: v[:, :nq] if v.ndim >= 2 else v
                    for k, v in outputs.items()
                    if k in ("pred_logits", "pred_boxes", "pred_masks", "pred_keypoints") and isinstance(v, Tensor)
                }
                results = self.postprocess(inference_outputs, orig_sizes)
            return {
                "loss": loss.detach() if self._use_manual_optimization else loss,
                "results": self._detach_results(results),
                "targets": targets,
            }
        return loss.detach() if self._use_manual_optimization else loss

    def _aux_aggregate_map(self, loss_dict: dict[str, Tensor], weight_dict: dict[str, float]) -> dict[str, str | None]:
        """Return the memoized ``loss_name -> aggregate train/ key`` map for the current loss_dict keys.

        The classification only depends on ``loss_dict``'s key set and ``weight_dict`` membership, both
        fixed by model architecture, so it is safe to compute once and cache across microbatches; it is
        recomputed only if the observed key set changes.

        Args:
            loss_dict: Per-term loss values for the current microbatch.
            weight_dict: Criterion-defined mapping of weighted loss-term names to their weights.

        Returns:
            Mapping from each ``loss_dict`` key to its ``train/<base>_aux`` aggregate key, or ``None``
            when the key should be logged individually.
        """
        keys = frozenset(loss_dict)
        if self._aux_aggregate_cache is None or self._aux_aggregate_cache_keys != keys:
            aggregate_map: dict[str, str | None] = {}
            for loss_name in loss_dict:
                base_name, separator, suffix = loss_name.rpartition("_")
                is_layer_suffixed = separator and (suffix.isdigit() or suffix == "enc")
                # Only weight_dict members are genuine per-layer loss terms; diagnostics such as
                # cardinality_error are never weighted, so they always fall through to individual logging.
                aggregate_map[loss_name] = (
                    f"train/{base_name}_aux" if is_layer_suffixed and loss_name in weight_dict else None
                )
            self._aux_aggregate_cache = aggregate_map
            self._aux_aggregate_cache_keys = keys
        return self._aux_aggregate_cache

    def _compact_train_loss_metrics(
        self, loss_dict: dict[str, Tensor], weight_dict: dict[str, float]
    ) -> dict[str, Tensor]:
        """Aggregate per-layer auxiliary loss terms into one ``train/<term>_aux`` tensor per base loss.

        Args:
            loss_dict: Per-term loss values for the current microbatch.
            weight_dict: Criterion-defined mapping of weighted loss-term names to their weights.

        Returns:
            Mapping of ``train/`` metric names to tensors, ready for ``self.log_dict``.
        """
        aggregate_map = self._aux_aggregate_map(loss_dict, weight_dict)
        grouped: dict[str, list[Tensor]] = {}
        train_loss_metrics: dict[str, Tensor] = {}
        for loss_name, value in loss_dict.items():
            aggregate_name = aggregate_map[loss_name]
            if aggregate_name is None:
                train_loss_metrics[f"train/{loss_name}"] = value
            else:
                grouped.setdefault(aggregate_name, []).append(value)
        for aggregate_name, values in grouped.items():
            train_loss_metrics[aggregate_name] = torch.stack(values).sum()
        return train_loss_metrics

    def _compute_train_losses(
        self,
        outputs: dict[str, Tensor],
        targets: list[dict[str, Tensor]],
    ) -> tuple[dict[str, Tensor], Tensor, Tensor]:
        """Compute normalized losses for logging and raw weighted loss for backward.

        Args:
            outputs: Model output dictionary.
            targets: Target dictionaries for the current batch.

        Returns:
            A tuple of normalized loss dictionary, unnormalized weighted loss numerator, and box normalizer.
        """
        weight_dict = self.criterion.weight_dict
        if not getattr(self.criterion, "supports_loss_normalizer_override", False):
            raise ValueError(
                f"{type(self.criterion).__name__}.supports_loss_normalizer_override is False; "
                "manual optimization (keypoint models) requires a criterion that accepts a "
                "num_boxes keyword argument. Set supports_loss_normalizer_override = True on "
                "your criterion subclass and implement the num_boxes parameter in forward()."
            )
        normalizer = self.criterion.num_boxes_for_targets(outputs, targets)
        numerator_loss_dict = self.criterion(outputs, targets, num_boxes=torch.ones_like(normalizer))
        # Keys in weight_dict are loss terms whose criterion implementation divides by num_boxes
        # (so passing num_boxes=1.0 yields raw numerators that we divide by normalizer here).
        # Keys outside weight_dict (e.g. "class_error", "cardinality_error") are diagnostics
        # that do NOT divide by num_boxes internally — they are passed through unchanged.
        # If a future loss term divides by num_boxes AND is omitted from weight_dict, its
        # logged value will be on a different scale than the keypoint path; verify when adding
        # new criterion terms.
        loss_dict = {
            key: value / normalizer if key in weight_dict else value for key, value in numerator_loss_dict.items()
        }
        raw_loss = sum(numerator_loss_dict[k] * weight_dict[k] for k in numerator_loss_dict if k in weight_dict)
        return loss_dict, raw_loss, normalizer

    def _scale_loss_for_accumulation(
        self,
        raw_loss: Tensor,
        normalizer: Tensor,
    ) -> Tensor:
        """Scale the current numerator loss by the accumulated box denominator.

        Args:
            raw_loss: Current microbatch weighted loss numerator.
            normalizer: Current microbatch box denominator.

        Returns:
            Loss scalar to pass to ``manual_backward``.
        """
        normalizer = normalizer.detach()
        previous_normalizer = self._accumulated_box_normalizer
        accumulated_normalizer = normalizer if previous_normalizer is None else previous_normalizer + normalizer
        if previous_normalizer is not None:
            self._rescale_accumulated_gradients(previous_normalizer / accumulated_normalizer)
        self._accumulated_box_normalizer = accumulated_normalizer.detach()
        return raw_loss / accumulated_normalizer

    def _rescale_accumulated_gradients(self, scale: Tensor) -> None:
        """Rescale gradients already accumulated in the current optimizer window.

        Args:
            scale: Multiplicative factor that converts previous gradients from the old denominator to the new one.
        """
        for parameter in self.parameters():
            if parameter.grad is not None:
                parameter.grad.mul_(scale.to(device=parameter.grad.device, dtype=parameter.grad.dtype))

    def _should_step_optimizer(self, batch_idx: int) -> bool:
        """Return whether the current batch closes an optimizer accumulation window.

        The optimizer steps when either:

        - The current batch closes a complete ``grad_accum_steps`` window
          (``(batch_idx + 1) % grad_accum_steps == 0``), or
        - This is the final batch of the epoch and a partial accumulation window
          is still open, so the trailing microbatches are not silently dropped.

        Lightning's ``Trainer.num_training_batches`` may be reported as ``float('inf')``
        for iterable / streaming datasets where the epoch length is unknown. In that case
        only the modulo path can ever close the window — the final-batch fallback is
        skipped because ``batch_idx + 1`` can never reach infinity.

        Args:
            batch_idx: Batch index within the epoch.

        Returns:
            ``True`` when the optimizer should step after this batch.
        """
        accum_steps = max(1, int(self.train_config.grad_accum_steps))
        if (batch_idx + 1) % accum_steps == 0:
            return True
        num_training_batches = getattr(self.trainer, "num_training_batches", None)
        return (
            isinstance(num_training_batches, (int, float))
            and math.isfinite(num_training_batches)
            and batch_idx + 1 >= num_training_batches
        )

    def _log_learning_rates(self, optimizer: torch.optim.Optimizer | LightningOptimizer) -> None:
        """Log the learning-rate range used by an optimizer update.

        Args:
            optimizer: Optimizer about to update model parameters.
        """
        group_lrs = [param_group["lr"] for param_group in optimizer.param_groups if "lr" in param_group]
        if not group_lrs:
            return
        self.log("train/lr", group_lrs[0], prog_bar=True, on_step=True, on_epoch=False)
        self.log("train/lr_min", min(group_lrs), prog_bar=False, on_step=True, on_epoch=False)
        self.log("train/lr_max", max(group_lrs), prog_bar=False, on_step=True, on_epoch=False)

    def _step_optimizer(self, optimizer: torch.optim.Optimizer | LightningOptimizer) -> None:
        """Clip gradients, step optimizer and scheduler, then reset accumulation state.

        Args:
            optimizer: Optimizer returned by Lightning.
        """
        trainer_gradient_clip_val = getattr(self.trainer, "gradient_clip_val", None)
        if trainer_gradient_clip_val is None:
            gradient_clip_val = self.train_config.clip_max_norm
        elif isinstance(trainer_gradient_clip_val, (int, float)):
            gradient_clip_val = trainer_gradient_clip_val
        else:
            gradient_clip_val = None
        gradient_clip_algorithm = getattr(self.trainer, "gradient_clip_algorithm", None)
        if not isinstance(gradient_clip_algorithm, str):
            gradient_clip_algorithm = None
        if gradient_clip_val is not None and gradient_clip_val > 0:
            self.clip_gradients(
                optimizer,
                gradient_clip_val=gradient_clip_val,
                gradient_clip_algorithm=gradient_clip_algorithm,
            )
        optimizer.step()
        optimizer.zero_grad()
        self._step_lr_scheduler()
        self._accumulated_box_normalizer = None

    def _current_lr_scheduler(self) -> LRScheduler | ReduceLROnPlateau | None:
        """Return the single configured LR scheduler, or ``None`` when none is available.

        Returns:
            The scheduler object (unwrapping Lightning's single-element list), or ``None`` when no
            scheduler is configured yet or Lightning is between fit stages.
        """
        try:
            scheduler = self.lr_schedulers()
        except (AttributeError, RuntimeError):
            return None
        if isinstance(scheduler, list):
            return scheduler[0] if scheduler else None
        return scheduler

    def _step_lr_scheduler(self) -> None:
        """Step step-interval schedulers once per optimizer step (manual-optimization path).

        Epoch-interval schedulers and metric-driven ``ReduceLROnPlateau`` are stepped at epoch boundaries by
        ``on_train_epoch_end`` / ``on_validation_epoch_end`` instead, so they are skipped here.
        """
        if self._lr_scheduler_interval != "step":
            return
        scheduler = self._current_lr_scheduler()
        if scheduler is None or isinstance(scheduler, ReduceLROnPlateau):
            return
        scheduler.step()

    def on_before_optimizer_step(self, optimizer: torch.optim.Optimizer) -> None:
        """Log rates immediately before each Lightning-managed optimizer step.

        Lightning invokes this hook for both automatic-optimization and
        manual-optimization (keypoint) training paths, so it is the sole
        emission site for learning-rate logging on either path.

        Args:
            optimizer: Optimizer about to update model parameters.
        """
        self._log_learning_rates(optimizer)

    def on_train_epoch_end(self) -> None:
        """Step epoch-interval (non-plateau) schedulers on the manual-optimization path.

        The automatic-optimization path leaves scheduler stepping entirely to Lightning; only the manual keypoint loop
        steps schedulers itself.
        """
        if self.automatic_optimization or self._lr_scheduler_interval != "epoch":
            return
        scheduler = self._current_lr_scheduler()
        if scheduler is None or isinstance(scheduler, ReduceLROnPlateau):
            return
        scheduler.step()

    def on_validation_epoch_start(self) -> None:
        """Resolve the validation-loss policy for this epoch, announcing an ``"auto"`` skip the first time it happens.

        The resolution is cached here — unconditionally, before every early return below — because ``validation_step``
        reads it on every batch while it depends only on configuration that cannot change mid-epoch.

        The consumer scan behind ``compute_val_loss="auto"`` only sees programmatic consumers — a plateau scheduler or a
        monitoring callback. It cannot see a human reading a ``val/loss`` curve out of ``metrics.csv``, TensorBoard, or
        Weights & Biases, who would otherwise find the curve silently gone. Stating the resolution once, on the first
        real validation epoch of the global-zero rank, gives that reader the knob to turn.
        """
        self._resolved_compute_val_loss = self._resolve_should_compute_val_loss()
        if self._logged_val_loss_skip_notice or self.train_config.compute_val_loss != "auto":
            return
        # The sanity-check pass runs before training and leaves the flag unset, so the first real epoch still logs.
        if getattr(self.trainer, "sanity_checking", False) or not getattr(self.trainer, "is_global_zero", True):
            return
        if self._resolved_compute_val_loss:
            return
        self._logged_val_loss_skip_notice = True
        logger.info(
            "Skipping validation-loss computation: compute_val_loss='auto' found no scheduler or callback monitoring "
            "'val/loss', so no 'val/loss' metric is logged this run. Set compute_val_loss=True to log it every "
            "validation epoch (for example to read the curve from metrics.csv, TensorBoard, or Weights & Biases)."
        )

    def on_validation_epoch_end(self) -> None:
        """Step ``ReduceLROnPlateau`` from the monitored metric on the manual-optimization path.

        The automatic-optimization path lets Lightning feed the monitored metric; the manual keypoint loop must read it
        from ``trainer.callback_metrics`` and step the scheduler itself. The pre-training sanity-check validation is
        skipped so plateau patience/cooldown bookkeeping is not seeded from the untrained model.
        """
        # build_trainer() defaults num_sanity_val_steps=0, so trainer.sanity_checking is False for
        # RFDETR.train() users by default; this half of the guard stays live only for direct
        # build_trainer() callers who explicitly re-enable sanity validation.
        if self.automatic_optimization or self.trainer.sanity_checking:
            return
        scheduler = self._current_lr_scheduler()
        if not isinstance(scheduler, ReduceLROnPlateau):
            return
        monitor = self._lr_scheduler_monitor or "val/loss"
        metric = self.trainer.callback_metrics.get(monitor)
        if metric is None:
            # Warn-and-continue would let the LR never reduce while training silently proceeds. Fail loud instead,
            # mirroring Lightning's strict-monitor behavior on the automatic-optimization path.
            raise RuntimeError(
                f"ReduceLROnPlateau monitor {monitor!r} was not found in callback_metrics, so the learning rate "
                "would never be reduced. Ensure the monitored metric is logged every validation epoch (e.g. set "
                "compute_val_loss=True for the default 'val/loss' monitor), or set lr_scheduler_monitor to a metric "
                "that is produced."
            )
        scheduler.step(metric)

    @staticmethod
    def _detach_results(results: list[dict[str, Tensor]]) -> list[dict[str, Tensor]]:
        """Detach postprocessed result tensors before handing them to callbacks.

        Args:
            results: Per-image postprocessed prediction dictionaries.

        Returns:
            Per-image dictionaries with tensor values detached from the graph.
        """
        return [
            {key: value.detach() if torch.is_tensor(value) else value for key, value in result.items()}
            for result in results
        ]

    def _log_train_progress_metrics(
        self,
        loss: Tensor,
        loss_dict: dict[str, Tensor],
        *,
        batch_size: int,
    ) -> None:
        """Log compact per-step convergence metrics for the progress bar only.

        Args:
            loss: Unscaled aggregate training loss.
            loss_dict: Raw criterion loss dictionary.
            batch_size: Current batch size used by Lightning for metric reduction metadata.
        """
        # When ``train_log_on_step`` is True, the ``train/loss`` call in ``training_step``
        # logs with ``on_step=True, on_epoch=True``; Lightning forks that into
        # ``train/loss_step`` + ``train/loss_epoch``, and ``train/loss_step`` already
        # provides the live per-step progress-bar view. Emitting this separate ``loss``
        # scalar in that case just duplicates it, so only log it on the default
        # ``train_log_on_step=False`` path.
        if not bool(self.train_config.train_log_on_step):
            self.log(
                "loss",
                loss,
                prog_bar=True,
                logger=False,
                on_step=True,
                on_epoch=False,
                batch_size=batch_size,
            )
        for loss_name, progress_name in _TRAIN_PROGRESS_LOSS_ALIASES.items():
            value = loss_dict.get(loss_name)
            if value is None:
                continue
            self.log(
                progress_name,
                value,
                prog_bar=True,
                logger=False,
                on_step=True,
                on_epoch=False,
                batch_size=batch_size,
            )

    def _log_val_loss_metrics(
        self,
        loss: Tensor,
        loss_dict: dict[str, Tensor],
        *,
        batch_size: int,
    ) -> None:
        """Log aggregate and component validation losses.

        Args:
            loss: Aggregate weighted validation loss.
            loss_dict: Raw criterion loss dictionary.
            batch_size: Current batch size used by Lightning for metric reduction metadata.
        """
        self.log_dict(
            {f"val/{k}": v for k, v in loss_dict.items()},
            on_step=False,
            on_epoch=True,
            sync_dist=True,
            batch_size=batch_size,
        )
        self.log("val/loss", loss, prog_bar=True, on_epoch=True, sync_dist=True, batch_size=batch_size)

    def _resolve_eval_model(self) -> Any:
        """Return the model to forward through for validation.

        Validation evaluates one model by default: the EMA-averaged weights when EMA is available,
        because those are the weights best-checkpoint selection ships. Forwarding through them here
        replaces the second, duplicate base+EMA forward pass ``COCOEvalCallback`` would otherwise run
        every validation batch (see issue 416). ``TrainConfig.eval_base_model`` opts back in to that
        comparison: the base model is forwarded here and ``COCOEvalCallback`` runs the EMA pass.

        Falls back to the base model when EMA is enabled but not yet warmed up (e.g. the very first
        validation epoch, before ``RFDETREMACallback.setup`` has built its averaged model), or when
        this module isn't attached to a ``Trainer`` at all (``LightningModule.trainer`` raises
        ``RuntimeError`` rather than returning ``None`` when unattached — e.g. ``validation_step``
        called directly outside ``Trainer.fit``/``Trainer.validate``). The same fallback covers
        ``use_ema=False``, where no EMA callback exists and the base model is the selected model.

        Uses the same ``_get_ema_inner_module`` helper as ``COCOEvalCallback`` (see
        ``coco_eval.py``) so both consumers resolve the EMA-averaged detection net through one
        shared code path instead of independently duck-typing ``RFDETREMACallback``.

        Returns:
            The EMA-averaged inner module's underlying detection net when it is the selected model and
            available, else the base model.
        """
        if self.train_config.eval_base_model:
            return self.model
        try:
            callbacks = getattr(self.trainer, "callbacks", [])
        except RuntimeError:
            return self.model
        for callback in callbacks:
            ema_inner = _get_ema_inner_module(callback)
            if ema_inner is not None:
                return ema_inner.model
        return self.model

    def validation_step(self, batch: tuple[Any, Any], batch_idx: int) -> dict[str, Any]:
        """Run forward pass and postprocess for one validation step.

        Returns raw results and targets so ``COCOEvalCallback`` can accumulate them across the epoch via
        ``on_validation_batch_end``.

        Args:
            batch: Tuple of (NestedTensor samples, list of target dicts).
            batch_idx: Batch index within the validation epoch.

        Returns:
            Dict with ``results`` (postprocessed predictions) and ``targets``.
        """
        samples, targets = batch
        if self._inductor_cudagraphs:
            # Same reason as in training_step: with eval_base_model=True or use_ema=False the
            # compiled model runs here and records its own eval-mode graph.
            torch.compiler.cudagraph_mark_step_begin()  # type: ignore[no-untyped-call]
        outputs = self._resolve_eval_model()(samples)
        if self._should_compute_val_loss:
            loss_dict = self.criterion(outputs, targets)
            weight_dict = self.criterion.weight_dict
            loss = sum(loss_dict[k] * weight_dict[k] for k in loss_dict if k in weight_dict)
            self._log_val_loss_metrics(loss, loss_dict, batch_size=len(targets))

        orig_sizes = torch.stack([t["orig_size"] for t in targets])
        results = self.postprocess(outputs, orig_sizes)
        return {"results": results, "targets": targets}

    @property
    def _validation_loss_is_monitored(self) -> bool:
        """Return whether a configured scheduler or callback consumes ``val/loss``.

        ``_lr_scheduler_monitor`` is set only after a concrete ``ReduceLROnPlateau`` scheduler is instantiated. Callback
        inspection keeps the ``"auto"`` policy compatible with PTL-native checkpoint or early stopping callbacks
        supplied through ``build_trainer(..., callbacks=...)``.

        RF-DETR's own callbacks are not covered by the PTL-native ``monitor`` attribute alone: ``BestModelCallback`` and
        ``RFDETREarlyStopping`` track two metrics at once and keep their real targets in ``_monitor_regular`` /
        ``_monitor_ema`` (``RFDETREarlyStopping.monitor`` is a synthetic key it injects itself), so all three attributes
        are inspected.
        """
        if self._lr_scheduler_monitor == "val/loss":
            return True
        try:
            callbacks = getattr(self.trainer, "callbacks", [])
        except RuntimeError:
            return False
        return any(
            "val/loss"
            in {
                getattr(callback, "monitor", None),
                getattr(callback, "_monitor_regular", None),
                getattr(callback, "_monitor_ema", None),
            }
            for callback in callbacks
        )

    def _resolve_should_compute_val_loss(self) -> bool:
        """Resolve whether validation should calculate loss for the current configuration.

        Returns:
            ``True`` when the loss is requested outright, or when the ``"auto"`` policy finds a ``val/loss`` consumer.
        """
        if self.train_config.compute_val_loss is True:
            return True
        return self.train_config.compute_val_loss == "auto" and self._validation_loss_is_monitored

    @property
    def _should_compute_val_loss(self) -> bool:
        """Return whether validation should calculate loss, reusing the resolution cached for this epoch.

        ``validation_step`` reads this once per batch while the ``"auto"`` policy resolution walks every configured
        callback, so ``on_validation_epoch_start`` resolves it once per validation epoch and caches the result here.
        The cache stays unset until that hook runs: a ``validation_step`` called directly, with no ``Trainer`` driving
        the loop, resolves the live configuration rather than reading a value frozen before the trainer was attached.
        """
        if self._resolved_compute_val_loss is not None:
            return self._resolved_compute_val_loss
        return self._resolve_should_compute_val_loss()

    @property
    def _fused_adamw_env_eligible(self) -> bool:
        """Return whether the runtime would enable fused AdamW, ignoring optimizer choice.

        Captures only the hardware/precision preconditions (BF16 on CUDA), so the
        custom-optimizer path can tell whether a dropped ``fused_optimizer=True``
        would actually have mattered.

        Returns:
            ``True`` when fused AdamW is requested and the runtime supports it.
        """
        return (
            self.model_config.fused_optimizer
            and torch.cuda.is_available()
            and torch.cuda.is_bf16_supported()
            and str(self.trainer.precision) in {"bf16-mixed", "bf16", "bf16-true", "transformer-engine"}
        )

    @property
    def _use_fused_optimizer(self) -> bool:
        """Return whether fused AdamW should be used for the current training configuration.

        Fused AdamW is only safe when the trainer's actual precision is a BF16 variant.  Checking GPU capability alone
        (``is_bf16_supported()``) is
        insufficient: on Ampere+ hardware that flag is always ``True`` even when
        the trainer is configured for ``32-true``, which causes a ``params, grads, exp_avgs, and exp_avg_sqs must have
        same dtype, device, and layout`` crash in DDP because gradient bucket views have non-matching strides in FP32.
        It additionally requires the built-in ``optimizer="adamw"`` selection: fused state normalization and gradient
        clipping are AdamW-specific and must not fire for custom optimizers.

        Returns:
            ``True`` when fused AdamW is requested, safe, and the built-in AdamW optimizer is selected.

        Examples:
            >>> from unittest.mock import patch
            >>> module = RFDETRModelModule.__new__(RFDETRModelModule)
            >>> module.model_config = type("Cfg", (), {"fused_optimizer": True})()
            >>> with patch("torch.cuda.is_available", return_value=False):
            ...     module._use_fused_optimizer
            False
        """
        if not self._fused_adamw_env_eligible:
            return False
        return _is_builtin_fused_adamw(self.train_config.optimizer)

    def configure_optimizers(self) -> OptimizerLRSchedulerConfig:
        """Build the configured optimizer with layer-wise LR decay and scheduler.

        Uses ``trainer.estimated_stepping_batches`` for total step count so cosine annealing covers the full training
        run regardless of dataset size or accumulation settings.
        ``optimizer="adamw"`` keeps RF-DETR's fused torch AdamW path;
        other names can be loaded from ``pytorch-optimizer``.

        Returns:
            PTL optimizer config dict with optimizer and step-interval scheduler.
        """
        tc = self.train_config
        ns = _namespace_from_configs(self.model_config, tc)

        # Unwrap torch.compile's OptimizedModule so get_param_dict sees the
        # original module's named_parameters() — compiled wrapper can cause
        # name-prefix mismatches that put the same tensor in multiple groups.
        model_for_params = getattr(self.model, "_orig_mod", self.model)
        param_dicts = get_param_dict(ns, model_for_params)

        optimizer_cfg = tc.optimizer
        optimizer: torch.optim.Optimizer
        if _is_builtin_fused_adamw(optimizer_cfg):
            # Built-in managed fused AdamW path (unchanged behavior).
            try:
                optimizer = torch.optim.AdamW(
                    param_dicts,
                    lr=tc.lr,
                    weight_decay=tc.weight_decay,
                    fused=self._use_fused_optimizer,
                    **tc.optimizer_kwargs,
                )
            except TypeError as exc:
                raise TypeError(
                    f"Failed to initialize optimizer 'adamw': {exc}. "
                    "Check optimizer_kwargs for arguments supported by torch.optim.AdamW."
                ) from exc
        else:
            if self._fused_adamw_env_eligible:
                logger.warning(_FUSED_IGNORED_MSG, optimizer_cfg)
            if not isinstance(optimizer_cfg, str):
                # Explicit callable / functools.partial: called with param groups only.
                callable_name = getattr(optimizer_cfg, "__qualname__", None) or repr(optimizer_cfg)
                optimizer = _instantiate_explicit_optimizer(optimizer_cfg, callable_name, param_dicts, {})
            elif _is_managed_optimizer_name(optimizer_cfg):
                # Managed native torch.optim short name (lr + signature-aware weight_decay injected).
                native_class: _OptimizerFactory = _resolve_native_optimizer(optimizer_cfg)
                optimizer = _instantiate_optimizer(native_class, optimizer_cfg, param_dicts, tc)
            else:
                # Explicit dotted import path: constructed from optimizer_kwargs only.
                optimizer_class = _import_optimizer_class(optimizer_cfg)
                optimizer = _instantiate_explicit_optimizer(
                    optimizer_class, optimizer_cfg, param_dicts, tc.optimizer_kwargs
                )

        # ``trainer.estimated_stepping_batches`` is reported in *microbatch* units when
        # the keypoint path runs with ``Trainer(accumulate_grad_batches=1)`` and manages
        # accumulation manually. ``LambdaLR.step()`` is called once per optimizer-step
        # (i.e. every ``grad_accum_steps`` microbatches), so the schedule must be sized
        # in optimizer-step units rather than microbatches; otherwise warmup and cosine
        # decay finish ``grad_accum_steps``× too early. Detection / segmentation models
        # still rely on Lightning's automatic optimization, where PTL already accounts
        # for ``accumulate_grad_batches`` inside ``estimated_stepping_batches`` and the
        # division below is a no-op (``grad_accum_steps`` would be 1 in that path).
        grad_accum_steps = max(1, int(tc.grad_accum_steps))
        microbatches = int(self.trainer.estimated_stepping_batches)
        # _should_step_optimizer steps the final partial window at epoch end, so the true
        # number of optimizer steps is ceil(microbatches / grad_accum_steps).  Using floor
        # would undercount when the epoch is not evenly divisible, causing warmup / cosine
        # schedules to finish one step earlier than the last actual step fires.
        total_steps = (
            max(1, math.ceil(microbatches / grad_accum_steps)) if self._use_manual_optimization else microbatches
        )
        steps_per_epoch = max(1, total_steps // tc.epochs)
        warmup_steps = int(steps_per_epoch * tc.warmup_epochs)

        scheduler_cfg = tc.lr_scheduler
        scheduler: LRScheduler | ReduceLROnPlateau
        interval = "step"
        monitor: str | None = None
        if _is_managed_scheduler_name(scheduler_cfg):
            # Managed "step" / "cosine" preset — warmup + total-step sizing baked into a LambdaLR (unchanged behavior).
            scheduler = _build_managed_scheduler(optimizer, tc, total_steps, steps_per_epoch, warmup_steps)
        else:
            if not isinstance(scheduler_cfg, str):
                # Explicit callable / functools.partial: built from the optimizer only (kwargs baked in).
                scheduler_name = getattr(scheduler_cfg, "__qualname__", None) or repr(scheduler_cfg)
                scheduler = _instantiate_explicit_scheduler(scheduler_cfg, scheduler_name, optimizer, {})
            else:
                # Explicit dotted import path: constructed from lr_scheduler_kwargs only.
                scheduler_class = _import_scheduler_class(scheduler_cfg)
                scheduler_kwargs = tc.lr_scheduler_kwargs
                # Only a per-parameter lr_lambda list needs the legacy unmerged groups;
                # regroup_unmerged_scheduler_kwargs returns its input untouched otherwise, so
                # skip rebuilding the (discarded) unmerged layout for every other scheduler.
                if isinstance(scheduler_kwargs.get("lr_lambda"), list):
                    scheduler_kwargs = regroup_unmerged_scheduler_kwargs(
                        scheduler_kwargs,
                        _build_param_dicts(ns, model_for_params),
                    )
                scheduler = _instantiate_explicit_scheduler(scheduler_class, scheduler_cfg, optimizer, scheduler_kwargs)
            interval = tc.lr_scheduler_interval
            if isinstance(scheduler, ReduceLROnPlateau):
                monitor = tc.lr_scheduler_monitor
                if monitor == "val/loss" and tc.compute_val_loss is False:
                    raise ValueError(
                        "compute_val_loss=False is incompatible with ReduceLROnPlateau monitoring 'val/loss'. "
                        "Set compute_val_loss=True or 'auto', or select a metric that is produced."
                    )
                # The monitored metric (e.g. val/loss) is only available per epoch, so plateau always steps
                # on the epoch boundary regardless of the configured interval.
                interval = "epoch"
                if warmup_steps > 0:
                    logger.warning(
                        "warmup_epochs=%s is ignored for ReduceLROnPlateau; a metric-driven scheduler cannot "
                        "be composed with a linear warmup ramp.",
                        tc.warmup_epochs,
                    )
            else:
                # Auto-wrap explicit schedulers with a linear warmup ramp. The wrap is stepped at the same cadence as
                # the scheduler, so size it in the scheduler's own units: optimizer steps for "step", epochs for "epoch"
                # (otherwise a step-sized ramp stepped once per epoch would stretch across the whole run).
                if interval == "step":
                    warmup_units = warmup_steps
                else:
                    # Epoch cadence: the ramp is stepped once per epoch. ceil keeps a fractional warmup_epochs from
                    # truncating to zero (a silently dropped warmup). A single-epoch ramp has start_factor == 1.0,
                    # i.e. a flat no-op that looks like warmup but isn't, so a gradual epoch-granular warmup needs
                    # >= 2 epochs; warn and skip rather than emit a degenerate ramp.
                    warmup_units = math.ceil(tc.warmup_epochs)
                    if tc.warmup_epochs > 0 and warmup_units < 2:
                        logger.warning(
                            "warmup_epochs=%s with lr_scheduler_interval='epoch' cannot form a gradual warmup ramp "
                            "(epoch-granular warmup needs >= 2 epochs); skipping warmup. Use "
                            "lr_scheduler_interval='step' for sub-epoch warmup, or set warmup_epochs >= 2.",
                            tc.warmup_epochs,
                        )
                        warmup_units = 0
                if warmup_units > 0:
                    scheduler = _wrap_with_warmup(scheduler, optimizer, warmup_units)

        self._lr_scheduler_interval = interval
        self._lr_scheduler_monitor = monitor

        lr_scheduler_config: LRSchedulerConfigType = {"scheduler": scheduler, "interval": interval}
        if monitor is not None:
            lr_scheduler_config["monitor"] = monitor
        return {
            "optimizer": optimizer,
            "lr_scheduler": lr_scheduler_config,
        }

    def clip_gradients(
        self,
        optimizer: torch.optim.Optimizer | LightningOptimizer,
        gradient_clip_val: float | None = None,
        gradient_clip_algorithm: str | None = None,
    ) -> None:
        """Override PTL gradient clipping to support fused AdamW.

        PTL's AMP precision plugin refuses to clip gradients when the optimizer declares it handles unscaling internally
        (fused=True).  When fused is active we are on BF16 (no GradScaler) so ``clip_grad_norm_`` is correct.  For the
        non-fused path (FP16 + GradScaler or FP32) we delegate to ``super()`` to preserve scaler-aware unscaling.

        Args:
            optimizer: The current optimizer.
            gradient_clip_val: Maximum gradient norm.
            gradient_clip_algorithm: Clipping algorithm; forwarded to super()
                for the non-fused path.
        """
        if self._use_fused_optimizer:
            if gradient_clip_val and gradient_clip_val > 0:
                torch.nn.utils.clip_grad_norm_(self.parameters(), gradient_clip_val)
        else:
            # PTL's own type stub only declares Optimizer here, but LightningOptimizer dynamically
            # multiply-inherits from the wrapped optimizer's class, so it satisfies this at runtime too.
            super().clip_gradients(
                optimizer,  # type: ignore[arg-type]
                gradient_clip_val=gradient_clip_val,
                gradient_clip_algorithm=gradient_clip_algorithm,
            )

    @staticmethod
    def _normalize_optimizer_state(optimizer: torch.optim.Optimizer) -> int:
        """Cast restored floating-point optimizer state tensors to match live parameters.

        Args:
            optimizer: AdamW optimizer whose state may have been rehydrated with a mismatched dtype or layout.

        Returns:
            Number of state tensors that were reallocated to match the current parameter layout.
        """
        normalized = 0
        for group in optimizer.param_groups:
            for param in group["params"]:
                state = optimizer.state.get(param)
                if not state:
                    continue
                for key, value in list(state.items()):
                    if not isinstance(value, torch.Tensor) or not value.is_floating_point():
                        continue
                    if value.shape != param.shape:
                        continue
                    if value.device == param.device and value.dtype == param.dtype and value.stride() == param.stride():
                        continue
                    restored = torch.empty_like(param)
                    restored.copy_(value.to(device=param.device, dtype=param.dtype))
                    state[key] = restored
                    normalized += 1
        return normalized

    def test_step(self, batch: tuple[Any, Any], batch_idx: int) -> dict[str, Any]:
        """Run forward pass and postprocess for one test step.

        Mirrors :meth:`validation_step` so ``COCOEvalCallback`` can accumulate results via ``on_test_batch_end`` when
        ``trainer.test()`` is called (e.g. from :class:`~rfdetr.training.callbacks.BestModelCallback` at end of
        training).

        Args:
            batch: Tuple of (NestedTensor samples, list of target dicts).
            batch_idx: Batch index within the test epoch.

        Returns:
            Dict with ``results`` (postprocessed predictions) and ``targets``.
        """
        samples, targets = batch
        outputs = self.model(samples)
        if self.train_config.compute_test_loss:
            loss_dict = self.criterion(outputs, targets)
            weight_dict = self.criterion.weight_dict
            loss = sum(loss_dict[k] * weight_dict[k] for k in loss_dict if k in weight_dict)
            self.log("test/loss", loss, sync_dist=True, batch_size=len(targets))

        orig_sizes = torch.stack([t["orig_size"] for t in targets])
        results = self.postprocess(outputs, orig_sizes)
        return {"results": results, "targets": targets}

    def predict_step(self, batch: tuple[Any, Any], batch_idx: int, dataloader_idx: int = 0) -> Any:
        """Run inference on a preprocessed batch and return postprocessed results.

        Args:
            batch: Tuple of (NestedTensor samples, list of target dicts).
            batch_idx: Batch index.
            dataloader_idx: Index of the predict dataloader.

        Returns:
            Postprocessed detection results from ``PostProcess``.
        """
        samples, targets = batch
        with torch.no_grad():
            outputs = self.model(samples)
        orig_sizes = torch.stack([t["orig_size"] for t in targets])
        return self.postprocess(outputs, orig_sizes)

    def on_load_checkpoint(self, checkpoint: dict[str, Any]) -> None:
        """Auto-detect legacy formats and reconcile PE shapes at checkpoint load time.

        PTL calls this hook before applying ``checkpoint["state_dict"]`` to the module.  Four normalisation steps are
        applied in order:

        1. **Raw legacy format** — a ``*.pth`` file loaded directly by
           ``Trainer`` (e.g. via ``ckpt_path=``).  Recognised by the presence of ``"model"`` without ``"state_dict"``.
           The state dict is rewritten in-place with the ``"model."`` prefix so PTL can apply it normally.

        2. **Positional-embedding interpolation** — when the checkpoint was
           saved at a different image resolution than the current model, the DINOv2 ``position_embeddings`` tensor shape
           will mismatch. :func:`~rfdetr.models.weights.interpolate_position_embeddings` is called to bicubic-resize the
           PE to ``model_config.positional_encoding_size`` before PTL applies the state dict.  Regression fix for
           :issue:`998`.

        3. **FP8 extra-state removal** — under Transformer Engine (``amp_dtype="fp8"``), the live module's
           ``state_dict()`` carries ``_extra_state`` entries recording FP8 scaling history. Since ``strict_loading``
           is disabled on this module, ``load_state_dict()`` tolerates a missing or unexpected key afterward, but
           that does not stop it from calling ``set_extra_state()`` for any such key present in both the checkpoint
           and the module —
           and Transformer Engine intentionally rejects pickle-deserialised extra state. The entries are removed from
           ``checkpoint["state_dict"]`` here so the full-checkpoint resume path (``Trainer(ckpt_path=...)``) cannot
           abort on them, mirroring :meth:`~rfdetr.training.callbacks.ema.RFDETREMACallback._without_extra_state`.

        4. **Converted format** — a file produced by
           :func:`~rfdetr.training.checkpoint.convert_legacy_checkpoint` that already has ``"state_dict"`` but also
           carries ``"legacy_ema_state_dict"``.  The EMA weights are stashed on ``self._pending_legacy_ema_state`` for
           optional restoration by :class:`~rfdetr.training.callbacks.ema.RFDETREMACallback`.

        Note:
            This hook only fires on ``Trainer(ckpt_path=...)`` resume paths. Fresh-train bootstrap from a
            ``pretrain_weights`` checkpoint runs through :func:`~rfdetr.models.weights.load_pretrain_weights` during
            ``__init__`` instead — that helper performs its own PTL ``.ckpt`` normalisation (``state_dict`` → ``model``
            key, ``_orig_mod`` strip) and PE interpolation, so the two code paths intentionally do not share state.

        Args:
            checkpoint: Checkpoint dict passed in by PTL (mutated in-place).
        """
        # Raw legacy .pth: no "state_dict" key — build it from "model".
        if "model" in checkpoint and "state_dict" not in checkpoint:
            checkpoint["state_dict"] = {"model." + k: v for k, v in checkpoint["model"].items()}

        # Interpolate DINOv2 positional embeddings when the checkpoint was saved
        # at a different resolution than the current model.  PTL applies
        # checkpoint["state_dict"] immediately after this hook, so the shapes
        # must already match at this point.  Regression: #998.
        if "state_dict" in checkpoint:
            interpolate_position_embeddings(
                checkpoint["state_dict"],
                self.model_config.positional_encoding_size,
            )

        # Drop Transformer Engine's `_extra_state` entries before PTL applies the state dict.
        # `strict_loading=False` (set in __init__) only tolerates a missing or unexpected key
        # afterward — it does not stop `load_state_dict()` from calling `set_extra_state()` for a
        # key present in both the checkpoint and the module, and Transformer Engine rejects that
        # pickle round-trip.
        if "state_dict" in checkpoint:
            extra_state_keys = [key for key in checkpoint["state_dict"] if key.rsplit(".", 1)[-1] == "_extra_state"]
            for key in extra_state_keys:
                del checkpoint["state_dict"][key]

        # Optimizer/scheduler state saved before parameters were grouped by hyperparameters carries
        # one parameter group per parameter, a layout the optimizer no longer has. Regroup it so
        # resuming such a run keeps its momentum and LR schedule instead of failing to load.
        regroup_unmerged_optimizer_state(checkpoint)

        # Stash legacy EMA weights for RFDETREMACallback.setup(), which restores
        # them into AveragedModel when resuming from converted legacy checkpoints.
        if "legacy_ema_state_dict" in checkpoint:
            self._pending_legacy_ema_state = checkpoint["legacy_ema_state_dict"]
            warnings.warn(
                "Checkpoint contains legacy EMA weights (`legacy_ema_state_dict`). "
                "Add RFDETREMACallback to your trainer callbacks to restore them; "
                "without it the stashed weights will be ignored.",
                UserWarning,
                stacklevel=2,
            )

    def reinitialize_detection_head(self, num_classes: int) -> None:
        """Reinitialize the detection head for a new class count.

        Args:
            num_classes: New number of classes (excluding background).
        """
        self.model.reinitialize_detection_head(num_classes)
