§
    WQßj  ã                   ó¸  — d Z ddlmZ ddlZddlZddlZddlZddlZ ee¦  «         	                    ¦   «         j
        d         Zej                             d ee¦  «        ¦  «         edz  Zd„ Zd„ Zd„ Zed	k    �r\ ej        e ¬
¦  «        Ze                     dg d¢¬¦  «         e                     dd¬¦  «         e                     dedg ¬¦  «         e                     ¦   «         Zej        rddlmZ ej        D ]Z ee¦  «         Œej        dk    r e¦   «          dS ej        dk    rddlm Z   e e¦  «         dS ej        dk    rBddl!m"Z" ddlm#Z#  e"¦   «         5   e#ej$        e¦  «         ddd¦  «         dS # 1 swxY w Y   dS ej        dk    r e% e¦   «         ¦  «        ‚ej        dk    rddl&m'Z'  e'edg¦  «         dS  e¦   «          dS dS )zEFresh M2-only retry version; no changes to pinned/running M1 workers.é    )ÚPathNé   z(outputs/pallet_mask_model_improvement_v4c                  óŠ  — dd l } ddlm} ddlm}m}m}m} t                               d¬¦  «          ||¦  «        \  }}t          |d         ¦  «        h d£z  }t          |¦  «        D ]„}	t          |	z  }
|
j                             ddd¬	¦  «         |                      ||	z  |
¦  «         |
                     d
¦  «          ||
¦  «         |||	z  ¦  «        k    rt          d|	z   ¦  «        ‚Œ… |t          ¦  «          |t          dz  t          dd ||dz  ¦  «         ||dz  ¦  «        ddddddgd¬¦
  «
        ¦  «         d S )Nr   )ÚOUTPUT)ÚreadÚwriteÚverify_datasetÚdigestiÀ  )ÚmodeÚfiles>   úresearch_permit.jsonúprivate/eligibility_audit.jsonlú private/preservation_before.jsonú"private/global_native_hashes.jsonlú$private/visual_review_decisions.jsonú&private/preservation_after_freeze.jsonúdataset_manifest.jsonT)r   ÚparentsÚexist_oki€  zCopy differs: zRETRY_SCOPE.jsonz<RF-DETR 1.11 scheduler option belongs in lr_scheduler_kwargsz8outputs/pallet_mask_model_improvement_v3/B0/failure.jsonzB0/failure.jsonr   Fz(outputs/pallet_mask_model_improvement_v3ÚB0ÚB1)
ÚreasonÚprevious_failed_B0Úprevious_failure_sha256Údataset_manifest_sha256Ú model_facing_data_byte_identicalÚtask_checkpoint_reusedÚM1_runs_remain_inÚ running_M1_code_or_data_modifiedÚM2_run_namesÚDB0_B1_model_loss_matcher_optimizer_ema_postprocess_configs_unchanged)ÚshutilÚ"pallet.mask_model_improvement_datar   Úpallet.mask_model_training_datar   r   r	   r
   ÚmkdirÚsetÚsortedÚparentÚcopyfileÚchmodÚ
ValueErrorÚdict)r"   Úsourcer   r   r	   r
   ÚmanifestÚ_ÚnamesÚnameÚdestinations              úU/root/private/project/pallet_detection_v2/scripts/run_mask_model_improvement_retry.pyÚpreparer4      sÉ  € Ø€M€M€MØCÐCÐCÐCÐCÐCØSÐSÐSÐSÐSÐSÐSÐSÐSÐSÐSÐSÝ
‡L‚L�e€LÑÔÐØ �. Ñ(Ô(�K€Hˆaõ �˜Ô!Ñ"Ô"ð &ið &ið &iñ i€Eõ �u‘”ð 4ð 4ˆÝ˜T‘kˆØÔ× Ò  e°TÀDÐ ÑIÔIÐIØ�Š˜˜t™ [Ñ1Ô1Ð1Ø×Ò˜%Ñ Ô Ð Øˆ6�+ÑÔ & &¨°©Ñ"5Ô"5Ò5Ð5ÝÐ-¨dÑ2Ñ3Ô3Ð3ð 6à€N•6ÑÔÐØ	€E�&Ð#Ñ
#¥TÐ1oØUØ &  vÐ.?Ñ'?Ñ @Ô @Ø &  vÐ.EÑ'EÑ FÔ FØ)-ÀeØDØ).¸dÀD¸\ØMQð&Sñ &Sô &Sñ Tô Tð Tð Tð Tó    c                  ó  — ddl m} m} ddg} |t          dz  t	          t          j        ¦   «         || d         d¬¦  «        ¦  «         g }|D �]“}t	          t
          j        | d         d	d	d
d¬¦  «        }t          |› d�z                       d¦  «        5 }t          j
        t          j        t          t          t          ¦  «                             ¦   «         ¦  «        d|g||t          j        ¬¦  «        } |t          |› d�z  t	          |j        || d         ¬¦  «        ¦  «         |                     ¦   «         }d d d ¦  «         n# 1 swxY w Y   |                     t	          ||¬¦  «        ¦  «         t+          t-          j        |d         ¦  «        d¬¦  «         |r= |t          dz  t	          d||t1          |¦  «        d …         ¬¦  «        ¦  «         |c S �Œ• |t          dz  t	          d|¬¦  «        ¦  «         dS )Nr   )Ú	GPU_UUIDSr   r   r   zqueue_m2_start.jsonÚm2T)ÚpidÚrunsÚcuda_visible_devicesÚfresh_generic_per_runÚ4Ú1z/tmp/opencode)ÚCUDA_VISIBLE_DEVICESÚOMP_NUM_THREADSÚMKL_NUM_THREADSÚPYTHONUNBUFFEREDÚTMPDIRz.logÚxbÚworker)ÚenvÚstdoutÚstderrz_process.json)r9   r1   r;   )r1   Ú
returncodeéÿÿÿÿ©Úflushzqueue_m2_completion.jsonÚfailed_evidence_retained)Ústater:   Ú	unstartedÚcompleted_all_bounded_runs)rN   r:   )r$   r7   r   r   r,   ÚosÚgetpidÚenvironÚopenÚ
subprocessÚPopenÚsysÚ
executableÚstrr   Ú__file__ÚresolveÚSTDOUTr9   ÚwaitÚappendÚprintÚjsonÚdumpsÚlen)	r7   r   r0   Ú	completedr1   rF   ÚstreamÚprocessÚcodes	            r3   Úqueuerg   +   sx  € Ø@Ð@Ð@Ð@Ð@Ð@Ð@Ð@Ø�4ˆL€EØ	€E�&Ð&Ñ
&­µ´±´À5Ø& tœ_ÀDð)Jñ )Jô )Jñ Kô Kð Kà€IØð ñ ˆÝ•2”:°I¸d´OØ#&¸ÈcÐZiðkñ kô kˆå˜�]�]�]Ñ"×(Ò(¨Ñ.Ô.ð 	"°&Ý Ô&­¬½½DÅ¹N¼N×<RÒ<RÑ<TÔ<TÑ8UÔ8UÐW_ÐaeÐ'fØ ­zÔ/@ðBñ Bô BˆGàˆE•&˜DÐ/Ð/Ð/Ñ/µ¸'¼+ÈDÐgpÐquÔgvÐ1wÑ1wÔ1wÑxÔxÐxØ—<’<‘>”>ˆDð		"ð 	"ð 	"ñ 	"ô 	"ð 	"ð 	"ð 	"ð 	"ð 	"ð 	"øøøð 	"ð 	"ð 	"ð 	"ð
 	×Ò� 4°DÐ9Ñ9Ô9Ñ:Ô:Ð:Ý�dŒj˜ 2œÑ'Ô'¨tÐ4Ñ4Ô4Ð4Øð 	ØˆE•&Ð3Ñ3µTÐ@ZÐajØ¥ I¡¤  Ô0ð62ñ 62ô 62ñ 3ô 3ð 3àˆKˆKˆKñ	ð 
€E�&Ð+Ñ
+­TÐ8TÐ[dÐ-eÑ-eÔ-eÑfÔfÐfØˆ1s   ÂB(E	Å	E	ÅE	c                  ó  — ddl m} m}m}m}m} ddlm}m} ddl	m
} i }|                     ¦   «         D �]2\  }	}
|
d         dk    r|nt          }||	z  dz  }|                     ¦   «         rš | |¦  «        } | |||	z  |d         z  ¦  «        |d	         k    d
¦  «         t          t          |                     t           ¦  «        ¦  «        | ||¦  «         | ||	z  dz  ¦  «         | ||	z  dz  ¦  «        ¬¦  «        ||	<   ŒÑ||	z  dz  }t          t          |                     t           ¦  «        ¦  «        d|                     ¦   «         r | |¦  «        nd ¬¦  «        ||	<   �Œ4 ||¦  «          |t          ¦  «          ||¦  «        }t          t#          d„ |                     ¦   «         D ¦   «         ¦  «        rdnd|| ||dz  ¦  «         ||dz  ¦  «         |t          dz  ¦  «        k    d dddddd¬¦  «        } |t          dz  |¦  «         t'          t)          j        t          |d         d„ |                     ¦   «         D ¦   «         d„ |                     ¦   «         D ¦   «         d¬¦  «        ¦  «        d¬¦  «         d S ) Nr   )r   r   r
   Úrequirer	   )r   Úaudit_after)ÚCONFIGSÚrouteÚm1zcompletion.jsonÚ
checkpointÚcheckpoint_sha256zCompleted checkpoint changedzdev_quality.jsonzcalibration_quality.json)ÚrootÚ
completionÚcompletion_sha256ÚdevÚcalibrationzfailure.jsonÚfailed_or_not_started)rp   rN   Úfailurec              3   ó   K  — | ]}d |v V — Œ	dS )rq   N© )Ú.0Úrs     r3   ú	<genexpr>zfinalize.<locals>.<genexpr>W   s)   è è € ÐFpÐFpÐ]^À|ÐWXÐGXÐFpÐFpÐFpÐFpÐFpÐFpr5   Ú%completed_bounded_screen_not_releasedÚpartial_or_failed_not_releasedr   Fz&v10 raw versions, not future latest GTTzUNo automatic A4/A5/new seeds; inspect complete group-stratified fixed-screen evidence)rN   r:   ÚpreservationÚimmutable_data_sha256Úfrozen_data_identicalÚaccepted_qualityÚproduction_deliveriesÚproduction_activationÚ formal_val_test_prediction_countr-   Úfailed_v3_B0_preservedÚcandidate_followupzFINAL_HANDOFF.jsonrN   c                 óh   — i | ]/\  }}||                      d i ¦  «                              d¦  «        “Œ0S )rq   Úactual_optimizer_updates©Úget©ry   ÚkÚvs      r3   ú
<dictcomp>zfinalize.<locals>.<dictcomp>^   sP   € ð  :Lð  :Lð  :LÑwzÐwxÐyz¸!¸A¿EºEÀ,ÈrÑ<RÔ<R×<VÒ<VÐWqÑ<rÔ<rð  :Lð  :Lð  :Lr5   c                 ó�   — i | ]C\  }}||                      d i ¦  «                              di ¦  «                              d¦  «        “ŒDS )rs   ÚallÚ
iou_medianr‰   r‹   s      r3   rŽ   zfinalize.<locals>.<dictcomp>_   sI   € ÐcÐcÐcÉsÈqÐQR˜˜1Ÿ5š5  r™?œ?×.Ò.¨u°RÑ8Ô8×<Ò<¸\ÑJÔJÐcÐcÐcr5   )rN   ÚupdatesÚdev_iou_mediansrƒ   rK   )r$   r   r   r
   ri   r	   r#   r   rj   Úpallet.mask_model_improvementrk   ÚitemsÚexistsr,   rY   Úrelative_toÚPROJECTr�   Úvaluesr_   r`   ra   )r   r   r
   ri   r	   rm   rj   rk   r:   r1   Úconfigrp   Úcompleterq   rv   r~   Úresults                    r3   Úfinalizer�   C   s�  € Ø\Ð\Ð\Ð\Ð\Ð\Ð\Ð\Ð\Ð\Ð\Ð\Ð\Ð\ØLÐLÐLÐLÐLÐLÐLÐLØ5Ð5Ð5Ð5Ð5Ð5Ø€DØŸš™œð Rñ R‰ˆˆfØ˜G”_¨Ò,Ð,ˆrˆrµ&ˆØ˜‘9Ð.Ñ.ˆØ�?Š?ÑÔð 		RØ˜˜h™œˆJØˆG�F�F˜4 ™9 Z°Ô%=Ñ=Ñ>Ô>À*ÐM`ÔBaÒað  dBñ  Cô  Cð  CÝ¥3 t×'7Ò'7½Ñ'@Ô'@Ñ#AÔ#AÈjØ"( &¨Ñ"2Ô"2¸¸¸TÀ$¹YÐGYÑ=YÑ8ZÔ8ZØ ˜D  d¡Ð+EÑ!EÑFÔFðHñ Hô HˆD�‰JˆJð ˜4‘i Ñ.ˆGÝ¥3 t×'7Ò'7½Ñ'@Ô'@Ñ#AÔ#AÐI`Ø6=·n²nÑ6FÔ6FÐ%P T T¨'¡]¤] ]ÈDðRñ Rô RˆD�‰J‰Jà€N�2ÑÔÐ˜˜¥vÑ.Ô.Ð.Ø�;˜r‘?”?€LÝÅ3ÐFpÐFpÐbf×bmÒbmÑboÔboÐFpÑFpÔFpÑCpÔCpð  WÐ?Ð?ð  wWØ ÀFÀFÈ2ÐNeÑKeÑDfÔDfØ$˜f RÐ(?Ñ%?Ñ@Ô@ÀFÀFÍ6ÐRiÑKiÑDjÔDjÒjØ°QÈeØ)*Ð3[Ø#ð  9PðQñ Qô Q€Fð 
€E�&Ð%Ñ
% vÑ.Ô.Ð.Ý	�$Œ*•T  w¤ð  :Lð  :Lð  C÷  Iò  Iñ  Kô  Kð  :Lñ  :Lô  :LØcÐcÐVZ×V`ÒV`ÑVbÔVbÐcÑcÔcð  |AðBñ Bô Bñ Cô Cð KOðPñ Pô Pð Pð Pð Pr5   Ú__main__)ÚdescriptionÚcommand)r4   Údeclarerg   rE   ú
wait-startr�   )Úchoicesr1   Ú?)Únargsz
--wait-pidr^   )ÚtypeÚactionÚdefault)Úwait_for_process_exitr4   r¡   )Ú
predeclarerE   )Úconfig_compatibility)rE   rg   r¢   )Úwait_receiptr   )(Ú__doc__Úpathlibr   Úargparser`   rQ   rU   rW   rZ   r[   r   r˜   ÚpathÚinsertrY   r   r4   rg   r�   Ú__name__ÚArgumentParserÚparserÚadd_argumentÚintÚ
parse_argsÚargsÚwait_pidÚ%verify_mask_model_training_history_v2r©   r9   r    r”   rª   Ú#pallet.mask_model_improvement_retryr«   rE   r1   Ú
SystemExitÚrun_mask_model_improvementr¬   rx   r5   r3   ú<module>r¾      s  ðØ KÐ KØ Ð Ð Ð Ð Ð Ø €€€Ø €€€Ø 	€	€	€	Ø Ð Ð Ð Ø 
€
€
€
à
ˆ$ˆx‰.Œ.×
 Ò
 Ñ
"Ô
"Ô
*¨1Ô
-€Ø „‡‚��3�3�w‘<”<Ñ  Ô  Ð  Ø	Ð;Ñ	;€ðTð Tð Tð:ð ð ð0Pð Pð Pð> ˆzÒÑØ$ˆXÔ$°Ð9Ñ9Ô9€FØ
×Ò˜	Ð+nÐ+nÐ+nÐÑoÔoÐoØ
×Ò˜ cÐÑ*Ô*Ð*Ø
×Ò˜¨3°xÈÐÑLÔLÐLØ×ÒÑÔ€DØ„}ð 'ØOÐOÐOÐOÐOÐOØ”=ð 	'ð 	'ˆCØ!Ð! #Ñ&Ô&Ð&Ð&Ø„|�yÒ Ð Øˆ‰	Œ	ˆ	ˆ	ˆ	Ø	Œ˜Ò	"Ð	"Ø<Ð<Ð<Ð<Ð<Ð<Øˆ
�6ÑÔÐÐÐØ	Œ˜Ò	!Ð	!ØLÐLÐLÐLÐLÐLØ8Ð8Ð8Ð8Ð8Ð8Ø!Ð!Ñ#Ô#ð 	&ð 	&ØˆF�4”9˜fÑ%Ô%Ð%ð	&ð 	&ð 	&ñ 	&ô 	&ð 	&ð 	&ð 	&ð 	&ð 	&ð 	&ð 	&øøøð 	&ð 	&ð 	&ð 	&ð 	&ð 	&à	Œ˜Ò	 Ð	 Øˆj˜˜™œÑ!Ô!Ð!Ø	Œ˜Ò	%Ð	%Ø;Ð;Ð;Ð;Ð;Ð;Øˆ�V˜d˜VÑ$Ô$Ð$Ð$Ð$àˆ‰
Œ
ˆ
ˆ
ˆ
ð5 Ðs   Å$FÆFÆ
F