{
  "state": "interrupted_hardware_failure",
  "stage": "training",
  "updated_at": "2026-10-10T09:29:01.079034+08:00",
  "scope": "human_annotation_assistance_only",
  "output": "/root/private/project/pallet_detection_v2/outputs/human_correction_manual_7086_v1",
  "completed_epochs": 120,
  "best_epoch": 102,
  "best_internal_validation_AP50_95": 0.9422647824590605,
  "first_ten_epochs_median_seconds": 91.42309440698591,
  "max_remaining_training_seconds": 16456.156993257464,
  "manual_merge": {
    "eligible_manual_frames": 7052,
    "all_checked_frames": 7086,
    "checked_external_excluded_from_fitting": 34,
    "new_source": {
      "archive": "/root/private/project/pallet_detection_v2/test_500.zip",
      "archive_sha256": "d2716a82fabc621eed9cafdc96e360c54d184266b19c7a66ab1d581da655469f",
      "frames": 500,
      "status_counts": {
        "pos": 488,
        "negative": 12
      },
      "boxes": 778,
      "geometry_unchanged": 263,
      "domains": {
        "224x172_fov_55_45": 12,
        "352x264_fov_40_30": 14,
        "640x480_fov_65_50": 94,
        "352x264_unverified": 96,
        "640x480_fov_80_60": 91,
        "352x264_fov_55_40": 114,
        "240x96_unverified": 64,
        "176x132_unverified": 15
      },
      "source_splits": {
        "test": 500
      },
      "scope": "human_annotation_assistance_only"
    },
    "training_frames": 6021,
    "validation_frames": 1031,
    "status_counts": {
      "train": {
        "pos": 5845,
        "negative": 176
      },
      "val": {
        "pos": 1001,
        "negative": 30
      }
    },
    "domain_counts": {
      "176x132_unverified": {
        "total_frames": 15,
        "target_frames": 2,
        "negative_frames": 0,
        "target_validation_negatives": 0,
        "negative_groups": 0,
        "negative_stratification_feasible": false,
        "validation_frames": 0,
        "validation_negatives": 0,
        "training_frames": 15,
        "training_negatives": 0,
        "validation_groups": 0,
        "training_groups": 1,
        "validation_unavailable_reason": "No positive group-disjoint holdout; all reviewed groups retained in auxiliary training."
      },
      "224x172_fov_55_45": {
        "total_frames": 488,
        "target_frames": 73,
        "negative_frames": 6,
        "target_validation_negatives": 1,
        "negative_groups": 1,
        "negative_stratification_feasible": false,
        "validation_frames": 73,
        "validation_negatives": 0,
        "training_frames": 415,
        "training_negatives": 6,
        "validation_groups": 7,
        "training_groups": 45
      },
      "240x96_unverified": {
        "total_frames": 64,
        "target_frames": 10,
        "negative_frames": 3,
        "target_validation_negatives": 0,
        "negative_groups": 1,
        "negative_stratification_feasible": false,
        "validation_frames": 0,
        "validation_negatives": 0,
        "training_frames": 64,
        "training_negatives": 3,
        "validation_groups": 0,
        "training_groups": 1,
        "validation_unavailable_reason": "No positive group-disjoint holdout; all reviewed groups retained in auxiliary training."
      },
      "352x264_fov_40_30": {
        "total_frames": 14,
        "target_frames": 2,
        "negative_frames": 0,
        "target_validation_negatives": 0,
        "negative_groups": 0,
        "negative_stratification_feasible": false,
        "validation_frames": 2,
        "validation_negatives": 0,
        "training_frames": 12,
        "training_negatives": 0,
        "validation_groups": 1,
        "training_groups": 1
      },
      "352x264_fov_55_40": {
        "total_frames": 2834,
        "target_frames": 425,
        "negative_frames": 41,
        "target_validation_negatives": 6,
        "negative_groups": 10,
        "negative_stratification_feasible": true,
        "validation_frames": 425,
        "validation_negatives": 6,
        "training_frames": 2409,
        "training_negatives": 35,
        "validation_groups": 17,
        "training_groups": 119
      },
      "352x264_unverified": {
        "total_frames": 1647,
        "target_frames": 247,
        "negative_frames": 65,
        "target_validation_negatives": 10,
        "negative_groups": 14,
        "negative_stratification_feasible": true,
        "validation_frames": 247,
        "validation_negatives": 10,
        "training_frames": 1400,
        "training_negatives": 55,
        "validation_groups": 24,
        "training_groups": 83
      },
      "640x480_fov_65_50": {
        "total_frames": 94,
        "target_frames": 14,
        "negative_frames": 0,
        "target_validation_negatives": 0,
        "negative_groups": 0,
        "negative_stratification_feasible": false,
        "validation_frames": 0,
        "validation_negatives": 0,
        "training_frames": 94,
        "training_negatives": 0,
        "validation_groups": 0,
        "training_groups": 1,
        "validation_unavailable_reason": "No positive group-disjoint holdout; all reviewed groups retained in auxiliary training."
      },
      "640x480_fov_80_60": {
        "total_frames": 1896,
        "target_frames": 284,
        "negative_frames": 91,
        "target_validation_negatives": 14,
        "negative_groups": 6,
        "negative_stratification_feasible": true,
        "validation_frames": 284,
        "validation_negatives": 14,
        "training_frames": 1612,
        "training_negatives": 77,
        "validation_groups": 8,
        "training_groups": 45
      }
    },
    "positive_boxes": {
      "train": 6644,
      "val": 1132
    },
    "remaining": {
      "test": 1849,
      "val": 1439
    },
    "scope": "human_annotation_assistance_only",
    "original_splits_preserved": true
  },
  "error": "User reports GPU0000:19:00.0 unknown device handle; RTX3090 disabled and requests fresh 4090D training",
  "superseded_by": "outputs/human_correction_manual_7086_v2_4090d"
}
