§
    fYßjú  ã                   ó´   — d Z ddlmZ ddlmZ ddlZddlZddlZddlZddl	m
Z
mZmZmZmZmZmZ ddlmZmZmZmZ e
dz  Ze
d	z  Zd
„ Zed„ ¦   «         Zd„ ZdS )zKExplicit post-fit device repair and B0 evidence recovery, no retraining B0.é    )Úcontextmanager)ÚPathNé   )ÚPROJECTÚreadÚwriteÚdigestÚrequireÚverify_datasetÚgpu_receipt)ÚGENERIC_SHAÚrf_itemsÚrf_predictionsÚreportz(outputs/pallet_mask_model_improvement_v4z(outputs/pallet_mask_model_improvement_v5c                 ó,   — |                       |¦  «        S )z=Lightning teardown may move the live module to CPU after fit.)Úto)ÚmodelÚdevices     úZ/root/private/project/pallet_detection_v2/pallet/mask_model_improvement_device_recovery.pyÚensure_inference_devicer      s   € à�8Š8�FÑÔÐó    c               #   ón   ‡K  — ddl m}  | j        Šˆfd„}|| _        	 d V — ‰| _        d S # ‰| _        w xY w)Nr   )Úmask_model_improvementc                 ó8   •—  ‰t          | d¦  «        g|¢R i |¤ŽS )Núcuda:0)r   )r   ÚargsÚkwargsÚoriginals      €r   Úon_gpuz,postfit_device_compatibility.<locals>.on_gpu   s.   ø€ ØˆxÕ/°°xÑ@Ô@ÐRÀ4ÐRÐRÐRÈ6ÐRÐRÐRr   )Ú r   r   )Úbaser   r   s     @r   Úpostfit_device_compatibilityr"      sw   øè è € à0Ð0Ð0Ð0Ð0Ð0ØÔ"€HðSð Sð Sð Sð Sà €DÔð'Øˆˆˆà&ˆÔÐÐø˜hˆÔÐ&Ð&Ð&Ð&s   ž+ «	4c                  ót  — ddl m}  ddlm} ddl}t          j        ¦   «         }t          dz  }|                     d¬¦  «         	 t          d	¦  «        }t          j        d
¦  «         t          j        d¦  «          |¦   «          t          t          ¦  «        \  }}t          t!          t          dz  ¦  «        t!          t"          dz  ¦  «        k    d¦  «         t"          dz  }t%          |dz  ¦  «        }	t          |	d         dk    o	d|	d         v d¦  «         t          |dz                       ¦   «          d¦  «         t          j        |dz  dd¬¦  «        }
t          |
d         dk    d¦  «         ~
t          j        |dz  dd¬¦  «        }t*          dz  }t          t!          |¦  «        t,          d	         k    d¦  «          | dd d!t/          |¦  «        dd"¬#¦  «        }t1          |j        j        d!¦  «        }|                     |d$         ¦  «         |d         }~t%          t          d%z  ¦  «        }t7          |d&z  t9          dGi d'd(“d)|“d*d+“d,t!          |dz  ¦  «        “d-t!          |d.z  ¦  «        “d/t!          |dz  ¦  «        “d0t!          |dz  ¦  «        “d1d“d2d“d3t!          t          dz  ¦  «        “d4|“d5d6“d7d"“d8d"“d9t!          t:          ¦  «        “d:t=          j        ¦   «         “Ž¦  «         d;D ]€}tA          |tC          t          |||¦  «        d¬<¦  «        \  }}t7          ||› d=�z  |d¬>¦  «         t7          ||› d?�z  tE          |¦  «        ¦  «         t7          |d@|› dA�z  |d¬>¦  «         Œ�t7          |dBz  t9          dCd+dd|t!          |dz  ¦  «        t!          |dz  ¦  «        t          j        ¦   «         |z
  dd"dd"¬D¦  «        ¦  «         dS # tF          $ rL}ddl$}t7          |dz  t9          dEt/          |¦  «        | %                    ¦   «         ¬F¦  «        ¦  «         ‚ d}~ww xY w)HzEEvaluate the already trained frozen best. ZERO new optimizer updates.r   )ÚRFDETRSegSmallr   )Úseed_allNÚB0_recoveryiÀ  )ÚmodeÚm2é   zdataset_manifest.jsonzRecovery data differÚB0zfailure.jsonÚ
error_typeÚRuntimeErrorz
Input typeÚerrorz!Unexpected recovery failure causezcompletion.jsonz(Do not overwrite/recover a completed runzlast.ptÚcpuT)Úmap_locationÚweights_onlyÚstepiè  z/B0 did not complete1000 actual training updateszbest.ptzNoutputs/pallet_mask_model_training_v1/generic_pretraining/rf-detr-seg-small.ptz+Generic architecture initialization changedi@  r   F)Únum_classesÚ
resolutionr   Úpretrain_weightsÚgradient_checkpointingÚcompiler   zprivate/rfdetr_transforms.jsonzrecovery_start.jsonÚstateÚ postfit_evaluation_recovery_onlyÚgpuÚoriginal_failed_runz+outputs/pallet_mask_model_improvement_v4/B0Úoriginal_failure_sha256Úoriginal_worker_start_sha256zworker_start.jsonÚsource_best_sha256Úsource_last_sha256Ú#trained_optimizer_updates_in_sourceÚnew_optimizer_updatesÚdataset_sha256Ú	best_stepÚfixed_native_thresholdg      à?Úselection_changedÚsource_run_modifiedÚrecovery_code_sha256Úpid)ÚdevÚcalibration)Ú
diagnosticz_predictions.jsonl)Úlinesz_quality.jsonÚB2_inference_only_z"_double_threshold_diagnostic.jsonlzrecovery_completion.jsonÚ6completed_postfit_evaluation_recovery_not_new_training)r7   Úoriginal_failed_training_rootÚ-actual_optimizer_updates_in_original_trainingr@   rB   r=   r>   Úelapsed_secondsÚoriginal_failure_preservedÚoriginal_run_completion_createdÚformal_val_test_predictionsÚproduction_activationÚrecovery_failed_not_complete)r7   r-   Ú	traceback© )&Úrfdetrr$   Úmask_model_trainingr%   ÚshutilÚtimeÚ	monotonicÚV5Úmkdirr   ÚtorchÚset_num_threadsÚcv2ÚsetNumThreadsr   r
   r	   ÚV4r   ÚexistsÚloadr   r   Ústrr   r   Úload_state_dictr   ÚdictÚ__file__ÚosÚgetpidr   r   r   ÚBaseExceptionrV   Ú
format_exc)r$   r%   rZ   ÚstartÚoutr9   Ú_ÚrecordsÚsourceÚfailureÚlastÚbestÚweightÚwrapperr   rB   Ú
transformsÚsplitÚpredictionsrJ   ÚexcrV   s                         r   Ú
recover_b0r|   #   sL  € à%Ð%Ð%Ð%Ð%Ð%Ø-Ð-Ð-Ð-Ð-Ð-Ø€M€M€MÝŒNÑÔ€EÝ
ˆ]Ñ
€CØ‡I‚I�5€IÑÔÐð.Ý˜$ÑÔˆÝÔ˜aÑ Ô Ð ¥#Ô"3°AÑ"6Ô"6Ð"6¸¸¹
¼
¸
Ý#¥BÑ'Ô'‰
ˆˆ7Ý••rÐ1Ñ1Ñ2Ô2µf½RÐ@WÑ=WÑ6XÔ6XÒXÐZpÑqÔqÐqÝ�D‘ˆÝ�v˜nÑ,Ñ-Ô-ˆÝ�˜Ô%¨Ò7Ð\¸LÈGÐT[ÔL\Ð<\ð  _Bñ  	Cô  	Cð  	CÝ�VÐ-Ñ-×5Ò5Ñ7Ô7Ð7Ð9cÑdÔdÐdÝŒz˜& Ñ*¸ÈTÐRÑRÔRˆÝ��V” Ò$Ð&WÑXÔXÐXØÝŒz˜& Ñ*¸ÈTÐRÑRÔRˆÝÐiÑiˆÝ•�v‘”¥+¨dÔ"3Ò3Ð5bÑcÔcÐcð !�.¨Q¸3ÀxÝ  ™[œ[ÀÈuðVñ Vô Vˆå'¨¬Ô(;¸XÑFÔFˆØ×Ò˜d 7œmÑ,Ô,Ð,Ø˜”Lˆ	 dÝ�"Ð=Ñ=Ñ>Ô>ˆ
ÝˆcÐ'Ñ'­ð *Dð *Dð *DÐ4VÐ4Vð *DÐ\_Ð\_ð *DØ MÐ Mð*Då$*¨6°.Ñ+@Ñ$AÔ$AÐ$Að*Dõ *0°Ð7JÑ0JÑ)KÔ)KÐ)Kð*Dõ  & f¨YÑ&6Ñ7Ô7Ð7ð	*Dõ MSÐSYÐZcÑScÑLdÔLdÐLdð	*Dð
 15°ð*Dð
 MNÈAð*Dõ "¥"Ð%<Ñ"<Ñ=Ô=Ð=ð*Dð JSÈð*Dð $& 2ð*Dð :?¸ð*Dð UZÐTYð*Dõ "(­Ñ!1Ô!1Ð!1ð*Dõ 8:´y±{´{°{ð*Dñ 	Eô 	Eð 	Eð ,ð 	nð 	nˆEÝ&4°U½HÅRÈÐRWÐYcÑ<dÔ<dÐquÐ&vÑ&vÔ&vÑ#ˆK˜Ý�#˜Ð2Ð2Ð2Ñ2°KÀtÐLÑLÔLÐLÝ�#˜Ð-Ð-Ð-Ñ-­v°kÑ/BÔ/BÑCÔCÐCÝ�#ÐT¨5ÐTÐTÐTÑTÐV`ÐhlÐmÑmÔmÐmÐmÝˆcÐ,Ñ,­dÐ9qØ*WØ:>ÐVWØµF¸6À)Ñ;KÑ4LÔ4LÕagÐhnÐoxÑhxÑayÔayÝ œNÑ,Ô,¨UÑ2ÈtØ,1ÈqÐhmð/oñ /oô /oñ 	pô 	pð 	pð 	pð 	pøõ ð ð ð ØÐÐÐÝˆc�.Ñ ¥$Ð-KÕSVÐWZÑS[ÔS[Ðgp×g{Òg{Ñg}Ôg}Ð"~Ñ"~Ô"~ÑÔÐØøøøøðøøøs   ÁNO! Ï!
P7Ï+AP2Ð2P7)Ú__doc__Ú
contextlibr   Úpathlibr   rj   r[   ra   r_   Úmask_model_training_datar   r   r   r	   r
   r   r   r   r   r   r   r   rc   r]   r   r"   r|   rW   r   r   ú<module>r�      s  ðØ QÐ QØ %Ð %Ð %Ð %Ð %Ð %Ø Ð Ð Ð Ð Ð Ø 	€	€	€	Ø €€€à 
€
€
€
Ø €€€à hÐ hÐ hÐ hÐ hÐ hÐ hÐ hÐ hÐ hÐ hÐ hÐ hÐ hÐ hÐ hÐ hÐ hØ QÐ QÐ QÐ QÐ QÐ QÐ QÐ QÐ QÐ QÐ QÐ QàÐ7Ñ7€ØÐ7Ñ7€ðð ð ð
 ð	'ð 	'ñ „ð	'ð6ð 6ð 6ð 6ð 6r   