§
    ðQßjU  ã                   ót   — d Z ddlmZ ddlZddlZddlZddlmZ ddl	m
Z
mZmZmZmZmZmZ ddlmZmZ d„ ZdS )	zJPost-queue independent CPU evidence; does not modify pinned training code.é    )ÚPathNé   )Únative_metrics)ÚPROJECTÚreadÚjsonlÚwriteÚdigestÚrequireÚverify_dataset)ÚCONFIGSÚGENERIC_SHAc                 ó*  ‡!— ddl m} i }| |fD ]}t          |¦  «         Œt          t	          | dz  ¦  «        t	          |dz  ¦  «        k    d¦  «         t          j        ¦   «         D �]ð\  }}|d         dk    r| n|}||z  }|dz                       ¦   «         st          d¬	¦  «        ||<   ŒFt          |dz  ¦  «        t          |d
z  ¦  «        }	}t          |d         dk    d¦  «         t          |	d         t          |d                  k    o|	d          d¦  «         t          |	d         t	          |dz  ¦  «        k    d¦  «         |	d                              ¦   «         D ]>\  }
}t          t	          |dz  t          |
¦  «        j        z  ¦  «        |k    d¦  «         Œ?||d         z  }t          t	          |¦  «        |d         k    d¦  «         t          j        |dd¬¦  «        }t          |d         |d         k    ot          |d         t          ¦  «        d¦  «         t!          d „ |d                              ¦   «         D ¦   «         ¦  «        }~t          j        |d!z  dd¬¦  «        }t          |d         dk    d"¦  «         ~d#„ t%          |d$z                       d%¦  «        ¦  «        D ¦   «         }t          t)          d&„ |D ¦   «         ¦  «        dk    d'¦  «         t          |d(z  ¦  «        }t          |d)         d*k    d+¦  «         d,„ t+          |d-z  ¦  «        D ¦   «         }i }d.D �]îŠ!t+          |‰!› d/�z  ¦  «        }ˆ!fd0„|                     ¦   «         D ¦   «         }t          t-          |¦  «        t-          |¦  «        k    od1„ |D ¦   «         |k    d2¦  «         d3}|D �]Ë}||d4                  }t          |d5         |d5         cxk    od6k    nc o|d7         |d7         k    o|d8         ‰!k    d9¦  «         |d:         }|                     t          |d;         |d<                              d=¦  «        ¬>¦  «        ¦  «                             t4          ¦  «        }t7          j        t;          ||d?         z  ¦  «        t6          j        ¦  «        dk    }t          |j        |j        k    d@¦  «         tA          ||¦  «        }|                     ¦   «         D ]s\  }}|�t          |t4          ¦  «        rt          |||         k    dA¦  «         Œ7tC          |||         z
  ¦  «        }t)          ||¦  «        }t          |dBk    dC¦  «         Œtt          |dD          o|dE          dF¦  «         �ŒÍt          |‰!› dG�z  ¦  «        }t          |dH         dk    o|dI         d u o|dJ         dk    dK¦  «         t          t-          |¦  «        t-          dL„ |D ¦   «         ¦  «        d|t	          |‰!› d/�z  ¦  «        t	          |‰!› dG�z  ¦  «        ¬M¦  «        |‰!<   �Œðt          dNdd|d         ||d         |t-          |¦  «        |¬O¦	  «	        ||<   �Œòt          dP|t	          tD          ¦  «        dQdd¬R¦  «        } tG          |dSz  | ¦  «         | S )TNr   )Úmaskzdataset_manifest.jsonzA/B frozen data differÚrouteÚm1zcompletion.jsonÚ"failed_or_not_started_not_complete)Ústatezworker_start.jsonÚactual_optimizer_updatesiè  zWrong actual update budgetÚgeneric_sha256Úold_finetuned_checkpoint_usedzContaminated pretrained startÚdataset_manifest_sha256zWorker data pin differsÚcode_sha256Úcode_snapshotzLaunch code snapshot changedÚ
checkpointÚcheckpoint_sha256zSelected checkpoint changedÚcpuT)Úmap_locationÚweights_onlyÚstepÚ	best_stepÚmodelzCPU checkpoint reload mismatchc              3   ór   K  — | ]2}t          |t          j        ¦  «        ¯|                     ¦   «         V — Œ3d S )N)Ú
isinstanceÚtorchÚTensorÚnumel)Ú.0Úvs     úS/root/private/project/pallet_detection_v2/pallet/mask_model_improvement_evidence.pyú	<genexpr>zverify.<locals>.<genexpr>$   s?   è è € Ð&qÐ&q°QÕU_Ð`aÕchÔcoÑUpÔUpÐ&q q§w¢w¡y¤yÐ&qÐ&qÐ&qÐ&qÐ&qÐ&qó    zlast.ptz%Last checkpoint update budget differsc                 ó,   — g | ]}t          |¦  «        ‘ŒS © )r   ©r(   Úps     r*   ú
<listcomp>zverify.<locals>.<listcomp>)   s   € ÐIÐIÐI˜a•$�q‘'”'ÐIÐIÐIr,   Úepochsz*.jsonc              3   ó&   K  — | ]}|d          V — ŒdS )Úoptimizer_stepNr.   )r(   Úes     r*   r+   zverify.<locals>.<genexpr>*   s(   è è € Ð8Ð8¨A�AÐ&Ô'Ð8Ð8Ð8Ð8Ð8Ð8r,   z)Epoch evidence lacks final optimizer stepz first_actual_training_batch.jsonr4   r   z*Actual first optimizer step receipt absentc                 ó    — i | ]}|d          |“ŒS ©Ú
target_keyr.   )r(   Úrs     r*   ú
<dictcomp>zverify.<locals>.<dictcomp>-   s   € Ð\Ð\Ð\¨!�1�\”? AÐ\Ð\Ð\r,   zprivate/training_records.jsonl)ÚdevÚcalibrationz_predictions.jsonlc                 ó2   •— h | ]\  }}|d          ‰k    ¯|’ŒS )Údevelopment_splitr.   )r(   Úkr9   Úsplits      €r*   ú	<setcomp>zverify.<locals>.<setcomp>1   s.   ø€ ÐYÐYÐY™d˜a ¸Ð;NÔ9OÐSXÒ9XÐ9X˜Ð9XÐ9XÐ9Xr,   c                 ó   — h | ]
}|d          ’ŒS r7   r.   r/   s     r*   rA   zverify.<locals>.<setcomp>2   s   € Ð:`Ð:`Ð:`Èq¸1¸\¼?Ð:`Ð:`Ð:`r,   z)All-target evaluation membership mismatchg        r8   Úformal_splitÚtrainÚ	componentr>   z&Evaluation leakage or stratum mismatchÚnative_prediction_rleÚsizeÚcountsÚascii)rG   rH   Únative_maskzNative prediction shape differsz!Native metric null/empty mismatchgê-�™—q=z$CPU native RLE metric replay differsÚground_truth_eligibleÚrobot_command_eligiblezPrediction auto-promotedz_quality.jsonÚaccepted_countÚaccepted_qualityÚ formal_val_test_prediction_countzUnsupported release claimc                 ó   — h | ]
}|d          ’ŒS )rE   r.   r/   s     r*   rA   zverify.<locals>.<setcomp>K   s8   € ð  ^Cð  ^Cð  ^CÐqrÐ^_Ð`kÔ^lð  ^Cð  ^Cð  ^Cr,   )Úall_outputsÚindependent_groupsÚnative_prediction_RLE_exactÚ&cpu_metric_maximum_absolute_differenceÚpredictions_sha256Úquality_sha256Ú!verified_completed_bounded_screen)	r   r   ÚCPU_checkpoint_reloadr   Ústate_tensor_elementsr!   Úactual_first_stepÚepoch_receiptsÚsplitsÚpassed_completed_run_evidenceF)r   ÚrunsÚverification_code_sha256Úold_files_modifiedÚval_test_predictionsÚno_production_activationz'CPU_TRAINING_EVIDENCE_VERIFICATION.json)$Úpycocotoolsr   r   r   r
   r   ÚitemsÚis_fileÚdictr   r   r   Únamer%   Úloadr$   ÚsumÚvaluesÚsortedÚglobÚmaxr   ÚlenÚdecodeÚencodeÚastypeÚboolÚcv2ÚimreadÚstrÚIMREAD_GRAYSCALEÚshaper   ÚabsÚ__file__r	   )"r   Úm2ÚmuÚresultsÚrootrg   ÚconfigÚoutÚcompleteÚstartÚpathÚexpectedr   r   Úselected_parameter_countÚlastr2   ÚfirstÚrecordsÚsplit_resultsÚpredictionsÚmaximumÚ
predictionÚrecordÚencodedr   ÚtargetÚmetricsÚkeyÚvalueÚ
differenceÚqualityÚresultr@   s"                                    @r*   Úverifyr•      s‘  ø€ Ø&Ð&Ð&Ð&Ð&Ð&Ø€GØ�R�ð ð ˆÝ�tÑÔÐÐÝ�F�2Ð-Ñ-Ñ.Ô.µ&¸Ð<SÑ9SÑ2TÔ2TÒTÐVnÑoÔoÐoÝœ™œð ?Wñ ?W‰ˆˆfØ˜G”_¨Ò,Ð,ˆrˆr°"ˆØ�4‰iˆØÐ%Ñ%×.Ò.Ñ0Ô0ð 	Ý Ð'KÐLÑLÔLˆG�D‰MØÝ˜sÐ#4Ñ4Ñ5Ô5µt¸CÐ@SÑ<SÑ7TÔ7T�%ˆÝ�Ð3Ô4¸Ò<Ð>ZÑ[Ô[Ð[Ý�Ð&Ô'­;°v¸g´Ô+GÒGð ;ØÐ9Ô:Ð:Ð<[ñ	]ô 	]ð 	]å�Ð/Ô0µF¸4Ð@WÑ;WÑ4XÔ4XÒXÐZsÑtÔtÐtØ# MÔ2×8Ò8Ñ:Ô:ð 	nð 	n‰NˆD�(Ý•F˜4 Ñ/µ°T±
´
´Ñ?Ñ@Ô@ÀHÒLÐNlÑmÔmÐmÐmØ˜ ,Ô/Ñ/ˆ
Ý•�zÑ"Ô" hÐ/BÔ&CÒCÐEbÑcÔcÐcÝ”
˜:°EÈÐMÑMÔMˆÝ��f” ¨+Ô!6Ò6Ð[½:ÀeÈGÄnÕVZÑ;[Ô;[Ð]}Ñ~Ô~Ð~Ý#&Ð&qÐ&q¸%À¼.×:OÒ:OÑ:QÔ:QÐ&qÑ&qÔ&qÑ#qÔ#qÐ ØÝŒz˜#˜i™-°eÈ$ÐOÑOÔOˆÝ��V” Ò$Ð&MÑNÔNÐNØØIÐI¥6¨3¨x©<×*=Ò*=¸hÑ*GÔ*GÑ#HÔ#HÐIÑIÔIˆÝ•Ð8Ð8°Ð8Ñ8Ô8Ñ8Ô8¸DÒ@ÐBmÑnÔnÐnÝ�SÐ;Ñ;Ñ<Ô<ˆÝ�Ð&Ô'¨1Ò,Ð.ZÑ[Ô[Ð[Ø\Ð\­u°TÐ:ZÑ5ZÑ/[Ô/[Ð\Ñ\Ô\ˆØˆØ+ð 	Dñ 	DˆEÝ  uÐ$@Ð$@Ð$@Ñ @ÑAÔAˆKØYÐYÐYÐY g§m¢m¡o¤oÐYÑYÔYˆHÝ•C˜Ñ$Ô$­¨H©¬Ò5ÐlÐ:`Ð:`ÐT_Ð:`Ñ:`Ô:`ÐdlÒ:lØ?ñAô Að AàˆGØ)ð 4ñ 4�
Ø  ¨LÔ!9Ô:�Ý˜˜~Ô.°*¸^Ô2LÐWÐWÒWÐWÐPWÒWÐWÐWÐWð AØ˜{Ô+¨z¸+Ô/FÒFðAà"Ð#6Ô7¸5Ò@ÐBjñlô lð lð %Ð%<Ô=�Ø—y’y¥¨7°6¬?À7È8ÔCT×C[ÒC[Ð\cÑCdÔCdÐ!eÑ!eÔ!eÑfÔf×mÒmÕnrÑsÔs�Ýœ¥C¨¨V°MÔ-BÑ(BÑ$CÔ$CÅSÔEYÑZÔZÐ]^Ò^�Ý˜œ
 f¤lÒ2Ð4UÑVÔVÐVÝ(¨¨vÑ6Ô6�Ø")§-¢-¡/¤/ð ]ð ]‘J�C˜Ø�}­
°5½$Ñ(?Ô(?�}Ý ¨°C¬Ò 8Ð:]Ñ^Ô^Ð^Ð^å%(¨¨z¸#¬Ñ)>Ñ%?Ô%?˜
Ý"% g¨zÑ":Ô":˜Ý 
¨eÒ 3Ð5[Ñ\Ô\Ð\Ð\Ý˜JÐ'>Ô?Ð?ÐlÈ
ÐSkÔHlÐDlØ2ñ4ô 4ð 4ñ 4å˜3 %Ð6Ð6Ð6Ñ6Ñ7Ô7ˆGÝ�GÐ,Ô-°Ò2ð E°wÐ?QÔ7RÐVZÐ7Zð EØÐ>Ô?À1ÒDÐFañcô cð cå#'µC¸Ñ4DÔ4DÕY\ð  ^Cð  ^Cð  wBð  ^Cñ  ^Cô  ^Cñ  ZDô  ZDØ,0ÐY`Ý#)¨#°Ð.JÐ.JÐ.JÑ*JÑ#KÔ#KÝ% c¨UÐ*AÐ*AÐ*AÑ&AÑBÔBð$Dñ $Dô $DˆM˜%Ñ Ñ õ Ð#FÐaeØ"&¸(ÐCVÔ:WØ":ÀhÈ{ÔF[Ø#µC¸±K´KÈðWñ Wô Wˆ�‰‰õ Ð7¸gÝ!'­Ñ!1Ô!1ÀeØ¸ð?ñ ?ô ?€Fõ 
ˆ"Ð6Ñ
6¸Ñ?Ô?Ð?Ø€Mr,   )Ú__doc__Úpathlibr   rs   ÚnumpyÚnpr%   Úmask_model_trainingr   Úmask_model_training_datar   r   r   r	   r
   r   r   Úmask_model_improvementr   r   r•   r.   r,   r*   ú<module>r�      s¹   ðØ PÐ PØ Ð Ð Ð Ð Ð à 
€
€
€
Ø Ð Ð Ð Ø €€€à /Ð /Ð /Ð /Ð /Ð /Ø bÐ bÐ bÐ bÐ bÐ bÐ bÐ bÐ bÐ bÐ bÐ bÐ bÐ bÐ bÐ bÐ bÐ bØ 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8Ð 8ðJð Jð Jð Jð Jr,   