§
    ‰ñÆjI…  ã                   óØ  — d Z ddlZddlmZmZ ddlZddlmZ ddlZddlZddl	Z	ddl
Z
ddlmZ ddlZddlmZ e
j                             dd¦  «         e
j                             d	d¦  «         ddlZdd
lmZ ddddddddddddœZd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Z d „ Z!d!„ Z"d2d"„Z#d3d#„Z$d$„ Z%d%„ Z&d4d'„Z'd(„ Z(d)„ Z)d*„ Z*d+„ Z+d,„ Z,d-„ Z-d.„ Z.d/„ Z/d0„ Z0e1d1k    r e0¦   «          dS dS )5a9  Compare saved proposals on a fixed, never manually reviewed frame cohort.

No training labels, independent reference boxes, or GPU inference are used.
Kept outside pallet/ so this analysis does not change a running trainer's source
fingerprint. Missing receipts are errors, never interpreted as empty detections.
é    N)ÚCounterÚdefaultdict)Údatetime)ÚPath)ÚZoneInfoÚOPENBLAS_NUM_THREADSÚ1ÚOMP_NUM_THREADS)Úlinear_sum_assignmentz>maximum cardinality one-to-one IoU>=0.50, then maximum IoU sumç      à?çffffffî?gÍÌÌÌÌÌì?g{®Gáz”?zLany added/deleted box or matched IoU<0.95 or edge shift>2% native short sidezXmaximum saved confidence floor across compared rounds; no lower candidates reconstructedzG2000 resamples of whole collection groups, frame-weighted ratio, seed42znboth adjacent pairs: <1% changed frames and <1% changed nonempty frames; both group bootstrap upper bounds <1%T)ÚmatchingÚ	match_iouÚ
stable_iouÚsensitivity_stable_iouÚmax_edge_shift_short_sideÚsubstantive_changeÚcommon_confidenceÚ	bootstrapÚoperational_convergenceÚno_ground_truth_scoringÚstable_does_not_mean_correctc                 óh   — t          j        t          | ¦  «                             ¦   «         ¦  «        S ©N)ÚjsonÚloadsr   Ú	read_text)Úpaths    ú%scripts/compare_proposal_stability.pyÚreadr    *   s$   € ÝŒ:•d˜4‘j”j×*Ò*Ñ,Ô,Ñ-Ô-Ð-ó    c                 ó  ‡— t          j        ¦   «         }t          | ¦  «                             d¦  «        5 Št	          ˆfd„d¦  «        D ]}|                     |¦  «         Œ	 d d d ¦  «         n# 1 swxY w Y   |                     ¦   «         S )NÚrbc                  ó.   •— ‰                       d¦  «        S )Ni  @ )r    )Ústreams   €r   ú<lambda>zdigest.<locals>.<lambda>1   s   ø€  &§+¢+¨kÑ":Ô":€ r!   r!   )ÚhashlibÚsha256r   ÚopenÚiterÚupdateÚ	hexdigest)r   ÚhÚblockr%   s      @r   Údigestr/   .   sÃ   ø€ ÝŒÑÔ€AÝ	ˆd‰Œ�Š˜Ñ	Ô	ð  &ÝÐ:Ð:Ð:Ð:¸CÑ@Ô@ð 	ð 	ˆEØ�HŠH�U‰OŒOˆOˆOð	ðð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð ð �;Š;‰=Œ=Ðs   ·,A0Á0A4Á7A4c                 óœ   — t          j        t          j        | dd¬¦  «                             ¦   «         ¦  «                             ¦   «         S )NTF)Ú	sort_keysÚensure_ascii)r'   r(   r   ÚdumpsÚencoder,   ©Úvalues    r   Újson_digestr7   6   s;   € ÝŒ>�$œ* U°dÈÐOÑOÔO×VÒVÑXÔXÑYÔY×cÒcÑeÔeÐer!   c                 ó  — t          | ¦  «        } | j                             dd¬¦  «         |                      | j        dz   ¦  «        }|                     t          j        |ddd¬¦  «        dz   ¦  «         |                     | ¦  «         d S )NT©ÚparentsÚexist_okz.tmpé   F)Úindentr2   Ú	allow_nanú
)	r   ÚparentÚmkdirÚwith_suffixÚsuffixÚ
write_textr   r3   Úreplace)r   r6   Ú	temporarys      r   ÚwriterG   :   sŠ   € Ý�‰:Œ:€DØ„K×Ò˜d¨TÐÑ2Ô2Ð2Ø× Ò  ¤¨vÑ!5Ñ6Ô6€IØ×Ò�œ E°!À%ÐSXÐYÑYÔYÐ\`Ñ`ÑaÔaÐaØ×Ò�dÑÔÐÐÐr!   c                 ó‚   — t          j        |                      ¦   «         ¦  «                             ¦   «         d d…         S )Né   )r'   r(   r4   r,   )Úfids    r   ÚslugrK   B   s/   € ÝŒ>˜#Ÿ*š*™,œ,Ñ'Ô'×1Ò1Ñ3Ô3°C°R°CÔ8Ð8r!   c                 ó(   — | st          |¦  «        ‚d S r   )Ú
ValueError)Ú	conditionÚmessages     r   ÚrequirerP   F   s"   € Øð "Ý˜Ñ!Ô!Ð!ð"ð "r!   c                 óü  — t          | dz  ¦  «        }t          |d         |k    d| › �¦  «         t          |d         ¦  «        t          |d         ¦  «        }}t          |d         ¦  «        }t          ||z   o||z  |k    d| › �¦  «         t          t          |¦  «        t          |d         ¦  «        k    d| › �¦  «         t          t          |d	         ¦  «        t          |d
         ¦  «        z   d| › �¦  «         |d                              ¦   «         D ]/\  }}t          t          | |z  ¦  «        |k    d| |z  › �¦  «         Œ0t          t          | dz  ¦  «        |d         k    d| › �¦  «         ||fS )Númanual_partition.jsonÚprotocol_sha256zPartition protocol mismatch: Útrain_frame_idsÚvalidation_frame_idsÚall_manual_frame_idszInvalid manual partition: zDuplicate manual IDs: Útrain_groupsÚvalidation_groupszGroup overlap: Úartifact_sha256zManual artifact changed: zinputs/annotations.jsonlÚmanual_annotations_sha256zManual label snapshot changed: )r    rP   ÚsetÚlenÚitemsr/   )r   Úprotocol_shaÚpÚtrainÚvalÚmanualÚnameÚexpecteds           r   Ú	partitionre   K   s«  € ÝˆTÐ+Ñ+Ñ,Ô,€AÝˆAÐÔ  LÒ0Ð2XÐRVÐ2XÐ2XÑYÔYÐYÝ�QÐ(Ô)Ñ*Ô*­C°Ð2HÔ0IÑ,JÔ,Jˆ3€EÝ�Ð)Ô*Ñ+Ô+€FÝ�˜‘ˆOÐ5 ¨¡¨vÒ 5Ð7ZÐTXÐ7ZÐ7ZÑ[Ô[Ð[Ý�C�‰KŒK�3˜qÐ!7Ô8Ñ9Ô9Ò9Ð;ZÐTXÐ;ZÐ;ZÑ[Ô[Ð[Ý•�A�nÔ%Ñ&Ô&­¨QÐ/BÔ-CÑ)DÔ)DÑDÐDÐF^ÐX\ÐF^ÐF^Ñ_Ô_Ð_ØÐ-Ô.×4Ò4Ñ6Ô6ð \ð \‰ˆˆhÝ•�t˜d‘{Ñ#Ô# xÒ/Ð1ZÈTÐTXÉ[Ð1ZÐ1ZÑ[Ô[Ð[Ð[Ý�F�4Ð4Ñ4Ñ5Ô5¸Ð;VÔ9WÒWØ4¨dÐ4Ð4ñ6ô 6ð 6àˆfˆ9Ðr!   c                 óè  ‡— i }| dz                        ¦   «                              ¦   «         D ]9}t          j        |¦  «        }t	          |d         |vd¦  «         |||d         <   Œ:t	          ‰t          |¦  «        k    d¦  «         ˆfd„|                     ¦   «         D ¦   «         }t	          |d¦  «         t	          t          d„ |D ¦   «         ¦  «        t          |¦  «        k    d¦  «         ||fS )	Núframes.jsonlÚframe_idzDuplicate source frame IDz'Manual frame missing from source exportc                 ó$   •— i | ]\  }}|‰v¯	||“ŒS © rj   )Ú.0rJ   ÚrowÚexcludeds      €r   ú
<dictcomp>zcohort.<locals>.<dictcomp>a   s)   ø€ ÐOÐOÐO™h˜c 3¸3ÀhÐ;NÐ;N��cÐ;NÐ;NÐ;Nr!   zEmpty comparison cohortc                 ó,   — h | ]}t          |¦  «        ’ŒS rj   ©rK   ©rk   rJ   s     r   ú	<setcomp>zcohort.<locals>.<setcomp>c   s   € Ð,Ð,Ð,˜s•�c‘”Ð,Ð,Ð,r!   zFrame filename hash collision)r   Ú
splitlinesr   r   rP   r[   r]   r\   )Úsourcerm   ÚindexÚlinerl   Ú	remainings    `    r   Úcohortrx   Z   sú   ø€ Ø€EØ˜.Ñ(×3Ò3Ñ5Ô5×@Ò@ÑBÔBð %ð %ˆÝŒj˜ÑÔˆÝ��J” uÐ,Ð.IÑJÔJÐJØ!$ˆˆc�*ŒoÑÐÝˆH�˜E™
œ
Ò"Ð$MÑNÔNÐNØOÐOÐOÐO¨%¯+ª+©-¬-ÐOÑOÔO€IÝˆIÐ0Ñ1Ô1Ð1Ý�CÐ,Ð, eÐ,Ñ,Ô,Ñ-Ô-µ°U±´Ò;Ð=\Ñ]Ô]Ð]Ø�)ÐÐr!   c                 ó”  — t          | j        dz  ¦  «        }t          | j        |¦  «        \  }}t	          | j        |¦  «        \  }}g g }}t          ¦   «         }| j        D �]'}	|	                     d¦  «        \  }
}}t          |o|
d¦  «         t          |¦  «         
                    ¦   «         }t          ||¦  «        \  }}t          ||cxk    o|k    nc d¦  «         t          |t          |¦  «        z   d¦  «         t          |t          |¦  «        k    d¦  «         |                     |
||f¦  «         |                     |
t          |¦  «        t          |¦  «        t          |dz  ¦  «        |d         d	œ¦  «         |}�Œ)t          t          t          d
„ |D ¦   «         ¦  «        ¦  «        t          |¦  «        k    d¦  «         t          t          |¦  «        dk    d¦  «         t          || j        t          | j        dz  ¦  «        t          |¦  «        t          |¦  «        t!          t#          d„ |                     ¦   «         D ¦   «         ¦  «        ¦  «        t'          t)          |¦  «        ¦  «        |t          | j        dz  ¦  «        t          | j        dz  ¦  «        t          t          t*          ¦  «        ¦  «        |d         dœ}| j                             dd¬¦  «         | j        dz  }|                     ¦   «         r"t          t3          |¦  «        |k    d¦  «         nt5          ||¦  «         ||||fS )Nzprotocol.jsonú=z!Use --round NAME=OUTPUT_DIRECTORYz/Manual cohorts must be chronological and nestedz4Manually reviewed frame entered stability comparisonz!Round contains unknown manual IDsrR   rZ   )rc   ÚoutputÚmanual_framesÚpartition_sha256Úmanual_snapshot_sha256c              3   ó"   K  — | ]
\  }}}|V — Œd S r   rj   )rk   rc   Ú_s      r   ú	<genexpr>zprepare.<locals>.<genexpr>z   s(   è è € Ð2Ð2™Z˜T 1 a�DÐ2Ð2Ð2Ð2Ð2Ð2r!   zDuplicate round namer<   zAt least two rounds requiredc              3   ó&   K  — | ]}|d          V — ŒdS ©ÚsplitNrj   ©rk   rl   s     r   r�   zprepare.<locals>.<genexpr>   s&   è è € Ð(TÐ(T¸#¨¨W¬Ð(TÐ(TÐ(TÐ(TÐ(TÐ(Tr!   zmanifest.jsonrg   )ÚmethodÚroundsÚexpected_roundsÚlatest_manual_partition_sha256Úexcluded_manual_framesÚremaining_framesÚsplit_countsÚremaining_frame_ids_sha256rS   Úsource_export_manifest_sha256Úsource_frame_index_sha256Úanalysis_source_sha256r~   Tr9   z	task.jsonz?Stability task input/method changed; use a new output directory)r/   Úprotocolre   Úmanual_outputrx   Úsource_exportr[   ÚroundrP   r   ÚresolveÚappendÚstrr\   ÚMETHODrˆ   Údictr   Úvaluesr7   ÚsortedÚ__file__r{   rA   Úexistsr    rG   )Úargsr^   Úlatest_partitionrm   ru   rw   r‡   ÚpartsÚpreviousÚspecificationrc   Ú	separatorÚlocationr   r_   ÚidsÚtaskÚ	task_paths                     r   Úpreparer¨   g   s0  € Ý˜$œ-¨/Ñ9Ñ:Ô:€LÝ!*¨4Ô+=¸|Ñ!LÔ!LÑÐ�hÝ˜dÔ0°(Ñ;Ô;Ñ€Eˆ9Ø˜ˆE€FÝ‰uŒu€HØœð ñ ˆØ$1×$;Ò$;¸CÑ$@Ô$@Ñ!ˆˆi˜Ý�	Ð"˜dÐ$GÑHÔHÐHÝ�H‰~Œ~×%Ò%Ñ'Ô'ˆÝ˜4 Ñ.Ô.‰ˆˆ3Ý�˜CÐ+Ð+Ò+Ð+ 8Ò+Ð+Ð+Ð+Ð-^Ñ_Ô_Ð_Ý�C�#˜i™.œ.Ñ(Ð(Ð*`ÑaÔaÐaÝ�•s˜5‘z”zÒ!Ð#FÑGÔGÐGØ�Š�t˜T 3Ð'Ñ(Ô(Ð(Ø�Š˜d­c°$©i¬iÍ#ÈcÉ(Ì(Ý*0°Ð8OÑ1OÑ*PÔ*PØ01Ð2MÔ0NðPð Pñ 	Qô 	Qð 	Qð ˆ‰Ý�C•Ð2Ð2¨6Ð2Ñ2Ô2Ñ2Ô2Ñ3Ô3µs¸6±{´{ÒBÐDZÑ[Ô[Ð[Ý�C�‰KŒK˜1ÒÐ<Ñ=Ô=Ð=Ý¨À$ÔBVÝ.4°TÔ5GÐJaÑ5aÑ.bÔ.bÝ&)¨(¡m¤mÍÈYÉÌÝ ¥Ð(TÐ(TÀ×AQÒAQÑASÔASÐ(TÑ(TÔ(TÑ!TÔ!TÑUÔUÝ*5µf¸YÑ6GÔ6GÑ*HÔ*HØ+Ý-3°DÔ4FÈÑ4XÑ-YÔ-YÝ)/°Ô0BÀ^Ñ0SÑ)TÔ)TÝ&,­Tµ(©^¬^Ñ&<Ô&<Ø&6Ð7RÔ&Sð	Uð 	U€Dð 	„K×Ò˜d¨TÐÑ2Ô2Ð2Ø”˜kÑ)€IØ×ÒÑÔð Ý•�Y‘” 4Ò'Ð)jÑkÔkÐkÐkåˆi˜ÑÔÐØ�5˜) TÐ)Ð)r!   c           	      ór  — |d         \  }}dD ];}t          |                      |¦  «        ||         k    d|› d|d         › �¦  «         Œ<t          |                      d¦  «        |                      d¦  «        f||fk    d¦  «         t          |                      d	¦  «        |k    d
¦  «         t          |                      d¦  «        |d         k    o|                      d¦  «        |d         k    d¦  «         t          |                      d¦  «        du o|                      d¦  «        du d¦  «         | d         }|rdnd}t          | d         |k    d¦  «         |D ]Õ}	|	d         }
t          t          |
¦  «        dk    o%t          j        |
¦  «                             ¦   «         d¦  «         |
\  }}}}t          d|cxk    o|cxk     o|k    nc od|cxk    o|cxk     o|k    nc d¦  «         t          t          j        |	d         ¦  «        o|dz
  |	d         cxk    odk    nc d ¦  «         ŒÖd S )!NÚsize)rh   Úgroup_idr„   ÚdomainÚsource_anchor_hashzReceipt z mismatch for rh   ÚwidthÚheightzReceipt dimensions changedÚbindingz#Receipt provenance binding mismatchÚcheckpoint_sha256Úthreshold_sha256zReceipt model/threshold changedÚhuman_reviewedFÚproposal_onlyTz&Receipt claims to be manually reviewedÚboxes_xyxy_scoreÚproposalÚzero_detection_unreviewedÚstatusz#Receipt status conflicts with boxesÚxyxyé   zInvalid box coordinatesr   zOut-of-bounds boxÚscoreg�íµ ÷Æ°>é   z+Box confidence below saved floor or invalid)rP   Úgetr\   ÚnpÚisfiniteÚall)rl   rt   r°   Úfloorr®   r¯   ÚkeyÚboxesÚexpected_statusÚboxr¹   Úx1Úy1Úx2Úy2s                  r   Úvalidate_receiptrÊ   �   sž  € Ø˜6”N�M€Eˆ6ØPð að aˆÝ�—’˜‘”  s¤Ò+Ð-_¸Ð-_Ð-_È6ÐR\ÔK]Ð-_Ð-_Ñ`Ô`Ð`Ð`ÝˆS�WŠW�WÑÔ˜sŸwšw xÑ0Ô0Ð1°e¸V°_ÒDÐFbÑcÔcÐcÝˆC�GŠG�IÑÔ 'Ò)Ð+PÑQÔQÐQÝˆC�GŠGÐ'Ñ(Ô(¨GÐ4GÔ,HÒHð KØ—’Ð*Ñ+Ô+¨wÐ7IÔ/JÒJÐLmñoô oð oåˆC�GŠGÐ$Ñ%Ô%¨Ð.ÐS°3·7²7¸?Ñ3KÔ3KÈtÐ3SØ4ñ6ô 6ð 6àÐ"Ô#€EØ$)ÐJ�j�jÐ/J€OÝˆC�ŒM˜_Ò,Ð.SÑTÔTÐTØð ?ð ?ˆØ�6Œ{ˆÝ•�D‘	”	˜Q’Ð:¥2¤;¨tÑ#4Ô#4×#8Ò#8Ñ#:Ô#:Ð<UÑVÔVÐVØ‰ˆˆB��BÝ��RÐ%Ð%Ò%Ð%˜"Ð%Ð%Ò%Ð% Ò%Ð%Ð%Ð%Ð@¨!¨rÐ*@Ð*@Ò*@Ð*@°BÐ*@Ð*@Ò*@Ð*@¸&Ò*@Ð*@Ð*@Ð*@ÐBUÑVÔVÐVÝ•”˜C œLÑ)Ô)ÐO¨e°d©l¸cÀ'¼lÐ.OÐ.OÒ.OÐ.OÈaÒ.OÐ.OÐ.OÐ.OØ=ñ	?ô 	?ð 	?ð 	?ð?ð ?r!   c                 óÎ  — t          |dz  ¦  «        }t          |d         dk    o|d         dk    o|d         |d         k    d¦  «         t          |¦  «        |z
  }t          |d         t          |¦  «        k    d	¦  «         d
„ |dz                       d¦  «        D ¦   «         }t          |d„ |D ¦   «         k    d¦  «         |d         }	dD ]%}
t          |	|
         ||
         k    d|
› �¦  «         Œ&|dz  }t          |¦  «        }t          |dz  ¦  «        }t          t          |dz  ¦  «        |d         cxk    o%|d         cxk    o|	d         cxk    o|d         k    nc d¦  «         t          t          |dz  ¦  «        |d         k    d¦  «         t          t          |¦  «        |	d         cxk    o|d         k    nc d¦  «         t          t          |dz  ¦  «        |	d         k    d¦  «         t          |d          d!u d"¦  «         t          |d#         ¦  «        }t          j        ¦   «         }i }t          |¦  «        D ]°}|dz  t          |¦  «        d$z   z  }|                     ¦   «         }t          j        |¦  «        }t          |||         |	|¦  «         |                     |                     ¦   «         d%z   t          j        |¦  «                             ¦   «         z   ¦  «         |||<   Œ±| t#          |¦  «        t          |¦  «        |d         t          |dz  ¦  «        |	d         ||                     ¦   «         t          |¦  «        |d&         d'œ
}||fS )(Núprediction_manifest.jsonÚstateÚcompleteÚerrorsr   Úcompleted_framesÚexpected_frameszPrediction not completezRound coverage count mismatchc                 ó   — h | ]	}|j         ’Œ
S rj   )rc   ©rk   r_   s     r   rr   zload_round.<locals>.<setcomp>«   s   € ÐGÐGÐG˜A�”ÐGÐGÐGr!   Úpredictionsz*.jsonc                 ó2   — h | ]}t          |¦  «        d z   ’ŒS )ú.jsonrp   rq   s     r   rr   zload_round.<locals>.<setcomp>¬   s#   € ÐBÐBÐB°#�$˜s™)œ) gÑ-ÐBÐBÐBr!   z!Missing/extra prediction receiptsr°   )rS   rŽ   r�   zRound source mismatch: zconfidence_threshold.jsonztraining/completed.jsonztraining/best.ptÚbest_checkpoint_sha256r±   zRound checkpoint mismatchztraining/run.jsonÚrun_config_sha256zRun config changedr²   zRound threshold mismatchrR   Úmanual_partition_sha256z(Round partition changed after predictionÚtest_accessedFzThreshold accessed test labelsÚselected_confidencerÖ   ó    Úcompleted_epochs)
rc   r{   r|   r±   Úprediction_manifest_sha256r²   Úconfidence_floorÚcohort_receipts_sha256Úcohort_framesrÝ   )r    rP   r[   r\   Úglobr/   Úfloatr'   r(   r›   rK   Ú
read_bytesr   r   rÊ   r+   r4   r—   r,   )rc   r   r¥   ru   rw   r¦   Úmanifestrd   Ú	filenamesr°   rÂ   Úthreshold_pathÚ	thresholdÚ
completionrÁ   Úreceipt_digestÚrowsrJ   ÚreceiptÚrawrl   Ú
provenances                         r   Ú
load_roundrï   ¥   sñ  € Ý�DÐ5Ñ5Ñ6Ô6€HÝˆH�WÔ Ò+ð L°¸Ô0BÀaÒ0Gð LØÐ+Ô,°Ð9JÔ0KÒKÐMfñhô hð hå�5‰zŒz˜CÑ€HÝˆHÐ'Ô(­C°©M¬MÒ9Ð;ZÑ[Ô[Ð[ØGÐG $¨Ñ"6×!<Ò!<¸XÑ!FÔ!FÐGÑGÔG€IÝˆIÐBÐB¸ÐBÑBÔBÒBÐDgÑhÔhÐhØ�yÔ!€GØ`ð Lð LˆÝ�˜”  S¤	Ò)Ð+JÀSÐ+JÐ+JÑKÔKÐKÐKØÐ7Ñ7€NÝ�^Ñ$Ô$€IÝ�dÐ6Ñ6Ñ7Ô7€JÝ�F�4Ð,Ñ,Ñ-Ô-°Ð<TÔ1Uð .ð .ò .ð .ØÐ+Ô,ð.ð .ò .ð .Ø07Ð8KÔ0Lð.ð .ò .ð .àÐ,Ô-ò.ð .ð .ð .à/JñLô Lð Lõ �F�4Ð-Ñ-Ñ.Ô.°*Ð=PÔ2QÒQÐSgÑhÔhÐhÝ�F�>Ñ"Ô" gÐ.@Ô&AÐaÐaÒaÐaÀXÐN`ÔEaÒaÐaÐaÐaØ&ñ(ô (ð (å�F�4Ð1Ñ1Ñ2Ô2°gÐ>WÔ6XÒXØ6ñ8ô 8ð 8åˆI�oÔ&¨%Ð/Ð1QÑRÔRÐRÝ�)Ð1Ô2Ñ3Ô3€EÝ”^Ñ%Ô%€NØ€DÝ�iÑ Ô ð ð ˆØ˜Ñ&­$¨s©)¬)°gÑ*=Ñ>ˆØ× Ò Ñ"Ô"ˆÝŒj˜‰oŒoˆÝ˜˜i¨œn¨g°uÑ=Ô=Ð=Ø×Ò˜cŸjšj™lœl¨UÑ2µW´^ÀCÑ5HÔ5H×5OÒ5OÑ5QÔ5QÑQÑRÔRÐRØˆˆS‰	ˆ	Ø­#¨d©)¬)ÅcÈ#ÁhÄhØ'/Ð0CÔ'DÝ06°tÐ>XÑ7XÑ0YÔ0YØ&-Ð.@Ô&AÐW\Ø,:×,DÒ,DÑ,FÔ,FÝ#& t¡9¤9À*ÐM_ÔB`ðbð b€Jð �ÐÐr!   c                 óö  — t          j        | t          ¬¦  «                             dd¦  «        t          j        |t          ¬¦  «                             dd¦  «        }} t          j        dt          j        | d d …d dd …f         |d d d …dd …f         ¦  «        t          j        | d d …d d d…f         |d d d …d d…f         ¦  «        z
  ¦  «        }|                     d¬¦  «        }| d d …dd …f         | d d …d d…f         z
                       d¬¦  «        }|d d …dd …f         |d d …d d…f         z
                       d¬¦  «        }|t          j        |d d …d f         |d d d …f         z   |z
  d¦  «        z  S )	N©Údtypeéÿÿÿÿrº   r   r<   )Úaxisr¼   gê-�™—q=)r¾   Úasarrayrã   ÚreshapeÚmaximumÚminimumÚprod)ÚaÚbÚextentÚoverlapÚarea_aÚarea_bs         r   Ú
iou_matrixr   Ï   s�  € ÝŒ:�a�uÐ%Ñ%Ô%×-Ò-¨b°!Ñ4Ô4µb´jÀÍ%Ð6PÑ6PÔ6P×6XÒ6XÐY[Ð]^Ñ6_Ô6_€q€AÝŒZ˜�2œ: a¨¨¨¨4°°°¨¤n°a¸¸a¸a¸aÀÀÀ¸´nÑEÔEÝœ* Q q q q¨$°°° {¤^°Q°t¸Q¸Q¸QÀÀÀ°{´^ÑDÔDñEñ Fô F€Fà�kŠk˜qˆkÑ!Ô!€GØ����1�2�2�Œh˜˜1˜1˜1˜b˜q˜b˜5œÑ!×'Ò'¨QÐ'Ñ/Ô/€FØ����1�2�2�Œh˜˜1˜1˜1˜b˜q˜b˜5œÑ!×'Ò'¨QÐ'Ñ/Ô/€FØ•R”Z  q q q¨$ w¤°&¸¸q¸q¸q¸´/Ñ AÀGÑ KÈUÑSÔSÑSÐSr!   c                 óÀ   ‡ ‡— ‰ j         sg S ‰ |k    Š‰t          ‰ j        ¦  «        dz   ‰ z   z  }t          |d¬¦  «        \  }}ˆˆ fd„t	          ||¦  «        D ¦   «         S )Nr¼   T)Úmaximizec           	      ó’   •— g | ]C\  }}‰||f         ¯t          |¦  «        t          |¦  «        t          ‰||f         ¦  «        f‘ŒDS rj   )Úintrã   )rk   rú   rû   ÚeligibleÚiouss      €€r   ú
<listcomp>zmatch_ious.<locals>.<listcomp>à   sP   ø€ Ð^Ð^Ð^±D°A°qÈxÐXYÐ[\ÐX\Ì~Ð^�S�‰VŒV•S˜‘V”V�U 4¨¨1¨¤:Ñ.Ô.Ð/Ð^Ð^Ð^r!   )rª   ÚminÚshaper   Úzip)r  rè   Úrewardsrë   Úcolsr  s   `    @r   Ú
match_iousr  Ù   st   øø€ ØŒ9ð Øˆ	à�yÒ €HØ�#˜dœj™/œ/¨AÑ-°Ñ4Ñ5€GÝ& w¸Ð>Ñ>Ô>�J€Dˆ$Ø^Ð^Ð^Ð^Ð^½CÀÀd¹O¼OÐ^Ñ^Ô^Ð^r!   c                 óT  ‡‡‡— ˆfd„} || ¦  «         ||¦  «        }}t          t          d„ |D ¦   «         d„ |D ¦   «         ¦  «        ¦  «        }g g g g f\  }	}
}}|D �]c\  }}}t          j        ||         d         ¦  «        t          j        ||         d         ¦  «        }}|	                     |¦  «         |
                     t          t          j        ||z
  ¦  «                             ¦   «         ¦  «        ¦  «         |                     t          t          j         	                    |d d…         |dd …         z   |d d…         z
  |dd …         z
  dz  ¦  «        t          j
        ‰‰¦  «        z  ¦  «        ¦  «         |                     t          ||         d         ||         d         z
  ¦  «        ¦  «         �Œeˆˆfd„|
D ¦   «         }t          |¦  «        t          |¦  «        z
  t          |¦  «        t          |¦  «        z
  }}t          d„ t          |	|¦  «        D ¦   «         ¦  «        }t          d	„ t          |	|¦  «        D ¦   «         ¦  «        }i d
t          |¦  «        “dt          |¦  «        “dt          |¦  «        “d|“d|“dt          | o| ¦  «        “dt          t          |p|¦  «        ¦  «        “dt          t          |¦  «        t          |¦  «        k    ¦  «        “dt          t          |¦  «        t          |¦  «        k    ¦  «        “dt          t          |p|¦  «        ¦  «        “d|“dt          |dk    ¦  «        “dt          t          |p|p|¦  «        ¦  «        “dt          t          |p|p|¦  «        ¦  «        “d|	“d|
“d|“||dœ¥S )Nc                 ó    •— ˆfd„| D ¦   «         S )Nc                 ó0   •— g | ]}‰�|d         ‰k    ¯|‘ŒS )Nr»   rj   )rk   rû   Ú
confidences     €r   r  z1compare_frame.<locals>.chosen.<locals>.<listcomp>å   s,   ø€ ÐSÐSÐS�a JÐ$6¸!¸G¼*È
Ò:RÐ:R�Ð:RÐ:RÐ:Rr!   rj   )rÃ   r  s    €r   Úchosenzcompare_frame.<locals>.chosenä   s   ø€ ØSÐSÐSÐS˜5ÐSÑSÔSÐSr!   c                 ó   — g | ]
}|d          ‘ŒS ©r¹   rj   ©rk   Úvs     r   r  z!compare_frame.<locals>.<listcomp>ç   s   € Ð"8Ð"8Ð"8° 1 V¤9Ð"8Ð"8Ð"8r!   c                 ó   — g | ]
}|d          ‘ŒS r  rj   r  s     r   r  z!compare_frame.<locals>.<listcomp>ç   s   € Ð:PÐ:PÐ:PÈ¸1¸V¼9Ð:PÐ:PÐ:Pr!   r¹   r<   r»   c                 ó6   •— g | ]}|t          ‰‰¦  «        z  ‘ŒS rj   )r  )rk   Úedger¯   r®   s     €€r   r  z!compare_frame.<locals>.<listcomp>ð   s(   ø€ ÐAÐAÐA°4�T�C  vÑ.Ô.Ñ.ÐAÐAÐAr!   c              3   ó`   K  — | ])\  }}|t           d          k     p|t           d         k    V — Œ*dS )r   r   N©r˜   ©rk   ÚiÚes      r   r�   z compare_frame.<locals>.<genexpr>ò   sT   è è € ð 9ð 9Ù�q˜!ð •v˜lÔ+Ò+ÐV¨qµ6Ð:UÔ3VÒ/Vð 9ð 9ð 9ð 9ð 9ð 9r!   c              3   ó`   K  — | ])\  }}|t           d          k     p|t           d         k    V — Œ*dS )r   r   Nr  r  s      r   r�   z compare_frame.<locals>.<genexpr>ô   sU   è è € ð ;ð ;Ù˜˜Að �Ð 8Ô9Ò9Ðd¸QÅÐHcÔAdÒ=dð ;ð ;ð ;ð ;ð ;ð ;r!   Ú	old_boxesÚ	new_boxesÚmatchedÚaddedÚdeletedÚ
both_emptyÚnonemptyÚpresence_changedÚcount_changedÚunmatched_changedÚgeometry_changed_boxesÚgeometry_changed_framesr   ÚchangedÚchanged_iou90r  Úedge_shift_pixelsÚedge_shift_short_side)Úcentre_shift_diagonalÚscore_difference)r  r   r¾   Úarrayr–   rã   ÚabsÚmaxÚlinalgÚnormÚhypotr\   Úsumr
  r  Úbool)ÚoldÚnewr®   r¯   r  r  rú   rû   Úpairsr  ÚedgesÚcentresÚscoresÚaiÚbirý   ÚaaÚbbÚedge_fractionr#  r$  ÚgeometryÚ
geometry90s     ```                  r   Úcompare_framerG  ã   s  øøø€ ðTð Tð Tð Tð Tàˆ6�#‰;Œ;˜˜˜s™œ€q€AÝ•zÐ"8Ð"8°aÐ"8Ñ"8Ô"8Ð:PÐ:PÈaÐ:PÑ:PÔ:PÑQÔQÑRÔR€EØ#% r¨2¨r >Ñ €Dˆ%�˜&Ø ð <ñ <‰ˆˆB�Ý”˜!˜Bœ% œ-Ñ(Ô(­"¬(°1°R´5¸´=Ñ*AÔ*AˆBˆØ�Š�GÑÔÐØ�Š•U�2œ6 " r¡'™?œ?×.Ò.Ñ0Ô0Ñ1Ô1Ñ2Ô2Ð2Ø�Š•u�RœYŸ^š^¨R°°°¬V°b¸¸¸´f©_¸rÀ"À1À"¼vÑ-EÈÈ1È2È2ÌÑ-NÐRSÑ,SÑTÔTÝ!œx¨¨vÑ6Ô6ñ7ñ 8ô 8ñ 	9ô 	9ð 	9à�Š•c˜!˜Bœ% œ.¨1¨R¬5°¬>Ñ9Ñ:Ô:Ñ;Ô;Ð;Ñ;ØAÐAÐAÐAÐA¸5ÐAÑAÔA€MÝ˜‘V”V�c %™jœjÑ(­#¨a©&¬&µ3°u±:´:Ñ*=ˆ7€EÝð 9ð 9Ý" 4¨Ñ7Ô7ð9ñ 9ô 9ñ 9ô 9€Håð ;ð ;Ý!$ T¨=Ñ!9Ô!9ð;ñ ;ô ;ñ ;ô ;€JðJˆK�˜Q™œð J ­c°!©f¬fð J°iÅÀUÁÄð JØ�UðJØ% wðJØ0<½cÀaÀ%À/ÐPQÈEÑ>RÔ>RðJà��D   a™LœLÑ)Ô)ðJà+=½sÅ4ÈÁ7Ä7ÍdÐSTÉgÌgÒCUÑ?VÔ?VðJð �S¥ Q¡¤­3¨q©6¬6Ò!1Ñ2Ô2ðJð 5HÍÍTÐRWÐRbÐ[bÑMcÔMcÑIdÔIdðJð % hð	Jð 1JÍ3ÈxÐZ[Ê|ÑK\ÔK\ð	Jð
 •s�4 Ð <¨Ð <°HÑ=Ô=Ñ>Ô>ðJð �S¥ eÐ&D¨wÐ&D¸*Ñ!EÔ!EÑFÔFðJð �DðJð .¨uðJð 7NÈ}ðJð &-À&ðJð Jð Jð Jr!   c                 ó   — |r| |z  nd S r   rj   )Ú	numeratorÚdenominators     r   ÚratiorK    s   € Ø&1Ð;ˆ9�{Ñ"Ð"°tÐ;r!   c           
      ó<  — | sdd d d d dœS t          | ¦  «        t          t          j        | ¦  «        ¦  «        t          t          j        | d¦  «        ¦  «        t          t          j        | ¦  «        ¦  «        t          t          j        | d¦  «        ¦  «        dœS )Nr   )ÚnÚmeanÚp05ÚmedianÚp95gš™™™™™©?r   )r\   rã   r¾   rN  ÚquantilerP  )rš   s    r   ÚdistributionrS    s‹   € Øð PØ ¨T¸TÈ$ÐOÐOÐOÝ�V‘”¥e­B¬G°F©O¬OÑ&<Ô&<Ý�œ V¨SÑ1Ô1Ñ2Ô2½eÅBÄIÈfÑDUÔDUÑ>VÔ>VÝ�œ V¨SÑ1Ô1Ñ2Ô2ð4ð 4ð 4r!   Fc           
      óv  ‡ ‡— ˆ fd„dD ¦   «         }t          ‰ ¦  «        t          d„ ‰ D ¦   «         ¦  «        dœ|¥t          |d         t          ‰ ¦  «        ¦  «        t          |d         |d         ¦  «        t          |d         t          ‰ ¦  «        ¦  «        t          |d         |d	         |d
         z   |d         z
  ¦  «        dœ¥}dD ] Št          ˆfd„‰ D ¦   «         ¦  «        |‰<   Œ!|�rÈt          t          ¦  «        }‰ D ]#}||d                                       |¦  «         Œ$t          |¦  «        dk     r
dddœ|d<   �nqt          j        d„ t          | 	                    ¦   «         ¦  «        D ¦   «         t          ¬¦  «        }t          j                             d¦  «        }|                     t          |¦  «        t          j        t          |¦  «        dt          |¦  «        z  ¦  «        d¬¦  «        }||z  }	|	d d …df         |	d d …df         z  }
|	|	d d …df         dk    df         |	|	d d …df         dk    df         z  }dddt          j        |
ddg¦  «                             ¦   «         t          |¦  «        r)t          j        |ddg¦  «                             ¦   «         nd dœ|d<   |S )Nc                 óJ   •‡— i | ]Š‰t          ˆfd „‰D ¦   «         ¦  «        “ŒS )c              3   ó(   •K  — | ]}|‰         V — Œd S r   rj   )rk   rl   rÂ   s     €r   r�   z'aggregate.<locals>.<dictcomp>.<genexpr>  s'   øè è € Ð0Ð0 C�s˜3”xÐ0Ð0Ð0Ð0Ð0Ð0r!   )r8  )rk   rÂ   rë   s    @€r   rn   zaggregate.<locals>.<dictcomp>  sK   øø€ ð að að a°Sˆc•3Ð0Ð0Ð0Ð0¨4Ð0Ñ0Ô0Ñ0Ô0ð að að ar!   )r   r!  r"  r#  r$  r%  r&  r'  r(  r)  r*  r+  r,  r-  c                 ó   — h | ]
}|d          ’ŒS )r«   rj   r…   s     r   rr   zaggregate.<locals>.<setcomp>  s   € Ð1RÐ1RÐ1RÀc°#°j´/Ð1RÐ1RÐ1Rr!   )ÚframesÚgroupsr,  r&  r-  r"  r   r!  )Úchanged_frame_rateÚchanged_nonempty_frame_rateÚchanged_frame_rate_iou90Úbox_set_jaccard)r  r.  r/  r0  r1  c                 ó*   •— g | ]}|‰         D ]}|‘ŒŒS rj   rj   )rk   rl   r6   rÂ   s      €r   r  zaggregate.<locals>.<listcomp>  s*   ø€ Ð#PÐ#PÐ#P¨cÀsÈ3ÄxÐ#PÐ#P¸e EÐ#PÐ#PÐ#PÐ#Pr!   r«   r<   Fzfewer than two groups)Ú	availableÚreasonÚgroup_bootstrap_95cic                 ó”   — g | ]E\  }}t          |¦  «        t          d „ |D ¦   «         ¦  «        t          d„ |D ¦   «         ¦  «        g‘ŒFS )c              3   ó&   K  — | ]}|d          V — ŒdS ©r,  Nrj   r  s     r   r�   z'aggregate.<locals>.<listcomp>.<genexpr>   s&   è è € Ð1NÐ1NÀ1°!°I´,Ð1NÐ1NÐ1NÐ1NÐ1NÐ1Nr!   c              3   ó&   K  — | ]}|d          V — ŒdS )r&  Nrj   r  s     r   r�   z'aggregate.<locals>.<listcomp>.<genexpr>   s*   è è € ÐSqÐSqÐfgÐTUÐV`ÔTaÐSqÐSqÐSqÐSqÐSqÐSqr!   )r\   r8  )rk   r€   r]   s      r   r  zaggregate.<locals>.<listcomp>   so   € ð !Ið !Ið !IÙ%- Q¨õ #& e¡*¤*­cÐ1NÐ1NÈÐ1NÑ1NÔ1NÑ.NÔ.NÕPSÐSqÐSqÐkpÐSqÑSqÔSqÑPqÔPqÐ!rð !Ið !Ið !Ir!   rñ   é*   r¼   iÐ  )rª   r   Tgš™™™™™™?g333333ï?)r_  Ú
replicatesÚseedrZ  r[  )r\   rK  rS  r   Úlistr–   r¾   rõ   r›   r]   rã   ÚrandomÚdefault_rngÚmultinomialÚfullrR  Útolist)rë   r   ÚtotalsÚresultrY  rl   ÚarraysÚrngÚweightsÚsumsÚratesÚactiverÂ   s   `           @r   Ú	aggregaterw    s  øø€ ðað að að að`ðañ aô a€Fõ ˜D™	œ	­SÐ1RÐ1RÈTÐ1RÑ1RÔ1RÑ-SÔ-Sð zð zÐW]ð zÝ$)¨&°Ô*;½SÀ¹Y¼YÑ$GÔ$GÝ-2°6¸)Ô3DÀfÈZÔFXÑ-YÔ-YÝ*/°°Ô0GÍÈTÉÌÑ*SÔ*SÝ!& v¨iÔ'8¸&ÀÔ:MÐPVÐWbÔPcÑ:cÐflÐmvÔfwÑ:wÑ!xÔ!xð	zð zð z€Fð
 sð Rð RˆÝ"Ð#PÐ#PÐ#PÐ#P°TÐ#PÑ#PÔ#PÑQÔQˆˆs‰ˆØñ tÝ�TÑ"Ô"ˆØð 	0ð 	0ˆCØ�3�z”?Ô#×*Ò*¨3Ñ/Ô/Ð/Ð/Ýˆv‰;Œ;˜Š?ˆ?Ø;@ÐLcÐ-dÐ-dˆFÐ)Ñ*Ñ*å”Zð !Ið !IÝ17¸¿º¹¼Ñ1GÔ1Gð!Iñ !Iô !IÝPUðWñ Wô WˆFå”)×'Ò'¨Ñ+Ô+ˆCØ—o’o¥c¨&¡k¤kµ2´7½3¸v¹;¼;ÈÍCÐPVÉKÌKÉÑ3XÔ3XÐ_c�oÑdÔdˆGØ˜VÑ#ˆDØ˜˜˜˜A˜”J  a a a¨ d¤Ñ+ˆEØ˜$˜q˜q˜q !˜tœ* qš.¨!Ð+Ô,¨t°D¸¸¸¸A¸´JÀ²NÀAÐ4EÔ/FÑFˆFà!°¸rÝ&(¤k°%¸$À¸Ñ&FÔ&F×&MÒ&MÑ&OÔ&OÝ]`ÐagÑ]hÔ]hÐ/r­r¬{¸6ÀDÈ$À<Ñ/PÔ/P×/WÒ/WÑ/YÔ/YÐ/YÐnrð.tð .tˆFÐ)Ñ*ð €Mr!   c                 ó>  — g }t          |                     ¦   «         ¦  «        D ]s\  }}t          | |         d         ||         d         g|d         ¢|‘R Ž }|                     ||d         |d         |d         ¬¦  «         |                     |¦  «         Œtdt          |d¬	¦  «        i}d
D ]m}	t          t          ¦  «        }
|D ]#}|
||	                                       |¦  «         Œ$d„ t          |
                     ¦   «         ¦  «        D ¦   «         |d|	z   <   Œn||fS )Nrµ   rª   r«   r¬   r„   )rh   r«   r¬   r„   ÚoverallT©r   )r„   r¬   c                 ó8   — i | ]\  }}|t          |d ¬¦  «        “ŒS )Trz  )rw  )rk   rc   r]   s      r   rn   zpair_result.<locals>.<dictcomp>:  s,   € ÐpÐpÐpÉ+È$ÐPU˜t¥Y¨uÀÐ%EÑ%EÔ%EÐpÐpÐpr!   Úby_)r›   r]   rG  r+   r–   rw  r   ri  )ÚleftÚrightrw   r  rë   rJ   rt   rl   rp  rÂ   rš   s              r   Úpair_resultr  .  sO  € Ø€DÝ˜iŸošoÑ/Ô/Ñ0Ô0ð ð ‰ˆˆVÝ˜D œIÐ&8Ô9¸5À¼:ÐFXÔ;Yð 9Ø# Fœ^ð9Ø-7ð9ð 9ð 9ˆà�
Š
˜C¨&°Ô*<ÀVÈHÔEUÐ]cÐdkÔ]lˆ
ÑmÔmÐmØ�Š�CÑÔÐÐØ� 4°4Ð8Ñ8Ô8Ð9€FØ"ð qð qˆÝ�TÑ"Ô"ˆØð 	)ð 	)ˆCØ�3�s”8Ô×#Ò# CÑ(Ô(Ð(Ð(ØpÐpÕY_Ð`f×`lÒ`lÑ`nÔ`nÑYoÔYoÐpÑpÔpˆˆu�s‰{ÑÐØ�4ˆ<Ðr!   c                 ó¢  — |sdd ddœS d„ | D ¦   «         }g }|D ]ƒ}|d         d         }|d         }|                      |d         d	k     oK|d
         d uoA|d
         d	k     o5|d         o-|d         d         d	k     o|d
         d uo|d
         d         d	k     ¦  «         Œ„t          |¦  «        ot          |¦  «        }|rdnd|dt          d         dœS )NÚpending_third_roundzQOnly the first pair is available; three-round convergence cannot yet be assessed.)rÍ   ÚstableÚexplanationc                 ó"   — g | ]}|d          ¯
|‘ŒS ©Úadjacentrj   ©rk   Úpairs     r   r  zjudgement.<locals>.<listcomp>B  s"   € Ð;Ð;Ð;˜¨$¨zÔ*:Ð;�Ð;Ð;Ð;r!   r   ry  ra  rZ  g{®Gáz„?r[  r_  r¼   Úmachine_proposals_stableÚmachine_proposals_not_convergedFr   )rÍ   r‚  Úcorrectness_confirmedÚoperational_rule)r–   r9  rÀ   r˜   )r<  rÎ   r†  Úpassingrˆ  ÚmetricsÚcir‚  s           r   Ú	judgementr�  >  s_  € Øð tØ.¸$Ørðtð tð 	tà;Ð; Ð;Ñ;Ô;€HØ€GØð Gð GˆØÐ*Ô+¨IÔ6ˆØÐ+Ô,ˆØ�Š�wÐ3Ô4°sÒ:ð FØ"Ð#@ÔAÈÐMðFà"Ð#@ÔAÀCÒGðFð ˜kœ?ðFð 02Ð2FÔ/GÈÔ/JÈSÒ/PðFð Ð;Ô<ÀDÐHð	Fð
 Ð;Ô<¸QÔ?À#ÒEñ	Gô 	Gð 	Gð 	Gõ �'‰]Œ]Ð+�s 7™|œ|€FØ39Ð`Ð/Ð/Ð?`Ø°uÝ &Ð'@Ô AðCð Cð Cr!   c                 ó   — | €dn| d›S )Nu   ä¸�å�¯è®¡ç®—z.2%rj   r5   s    r   Úpercentr’  S  s   € Ø"˜]ˆ>ˆ>°5°°Ð>r!   c                 óœ  — ddd| d         dk    rdnd› d�dd	| d
         d         d›d| d
         d         d›d�dddddddd| d         d›d�dddddddg}| d         D ]½}|d         d         }|d         d         €dn|d         d         d ›}|                      d!|d"         › d#|d$         › d%|d&         › d'|d(         › d)t          |d*         ¦  «        › d+t          |d,         ¦  «        › d%|d-         › d%|d.         › d'|d/         › d%|› d%|d0         › d1�¦  «         Œ¾|g d2¢z  }| d         D ]²}|d         d         }|d3         }|d4         sdn$d5                     d6„ |d*         D ¦   «         ¦  «        }|d7         }|d         €dn|d         d›d'|d8         d›�}	|                      d!|d"         › d#|d$         › d%|› d%t          |d9         ¦  «        › d%|	› d1�¦  «         Œ³|g d:¢z  }| d;         }
| d         dk    r|                      d<¦  «         n3|
d=         r|                      d>¦  «         n|                      d?¦  «         t          | d         ¦  «        d@k    rXdA„ | d         D ¦   «         \  }}||k     rdBndC}|                      dDt          |¦  «        › dEt          |¦  «        › dF|› dG�¦  «         |g dH¢z  }dI                     |¦  «        S )JNu5   # æœªäººå·¥æ£€æŸ¥å€™é€‰æ ‡æ³¨çš„ä¸‰è½®ç¨³å®šæ€§æ¯”è¾ƒÚ u   çŠ¶æ€�ï¼š**rÍ   rÎ   u   å®Œæ•´ä¸‰è½®æ¯”è¾ƒu0   ä»…å‰�ä¸¤è½®åˆ�æ­¥æ¯”è¾ƒï¼Œç¬¬ä¸‰è½®å°šæœªå®Œæˆ�u   **ã€‚u   å›ºå®šæ¯”è¾ƒ rx   r‹   ú,u    å¸§ï¼ŒæŽ’é™¤ç´¯è®¡ rŠ   u     å¼ äººå·¥æ£€æŸ¥å¸§ï¼ˆå�«äººå·¥éªŒè¯�å¸§ï¼‰ã€‚æ‰€æœ‰è½®æ¬¡å�‡æœªä»¥è¿™äº›å¸§çš„äººå·¥æ ‡ç­¾è®­ç»ƒæˆ–æ ¡å‡†ï¼›åŽŸæœºå™¨æ ‡ç­¾å’Œç‹¬ç«‹å�‚è€ƒæ¡†æœªå�‚ä¸Žè¯„åˆ†ã€‚u»   æ¯”è¾ƒçš„æ˜¯æ¨¡åž‹å»ºè®®çš„ä¸€è‡´ç¨‹åº¦ï¼Œä¸�èƒ½ç¡®è®¤æ¼�æ£€ã€�å…±å�Œè¯¯æ£€æˆ–å…±å�Œé”™è¯¯æ¡†ã€‚ä¸‰è½®è®­ç»ƒæ•°æ�®é‡�å�Šç¬¬ä¸‰è½®å®žé™… batch ä¸�å�Œï¼Œå�˜åŒ–ä¸�èƒ½å½’å› äºŽå�•ä¸€å› ç´ ã€‚u   ## å›ºå®šæ¯”è¾ƒè§„åˆ™u  ä»¥ IoUâ‰¥0.50 å�šä¸€å¯¹ä¸€åŒ¹é…�ï¼Œä¼˜å…ˆæœ€å¤šåŒ¹é…�ï¼Œå†�æœ€å¤§åŒ– IoUã€‚æ–°å¢ž/æ¶ˆå¤±æ¡†ã€�åŒ¹é…� IoU<0.95 æˆ–ä»»ä¸€è¾¹ç•Œç§»åŠ¨è¶…è¿‡åŽŸç”Ÿå›¾çŸ­è¾¹2%ï¼Œå�‡ä¸ºå®žè´¨å�˜åŒ–ã€‚å�¦æŠ¥å‘Š IoU=0.90 çš„æ•�æ„Ÿæ€§ç»Ÿè®¡ã€‚ç©ºæ¡†ä¸Žç©ºæ¡†å�•ç‹¬è®¡æ•°ï¼Œä¸�å½“ä½œæ­£ç¡®è´Ÿä¾‹ã€‚u   å…±å�Œç½®ä¿¡åº¦é˜ˆå€¼ä¸º **r   ú.2fu•   **ï¼ˆå�„è½®ä¿�å­˜é˜ˆå€¼çš„æœ€å¤§å€¼ï¼‰ï¼Œå�Œæ—¶ä¿�ç•™å�„è‡ªäº¤ä»˜é˜ˆå€¼çš„æ¯”è¾ƒã€‚æ—§è½®é˜ˆå€¼ä»¥ä¸‹å€™é€‰æ— æ³•ä»Žä¿�å­˜æ–‡ä»¶æ�¢å¤�ï¼Œæœªè‡†é€ ã€‚u!  95%åŒºé—´æŒ‰å®Œæ•´é‡‡é›†ç»„é‡�é‡‡æ ·2,000æ¬¡ã€�seed42ï¼Œç»Ÿè®¡å¸§åŠ æ�ƒå�˜åŒ–çŽ‡ã€‚æœ¬è½®é¢„å…ˆç™»è®°çš„æ“�ä½œæ€§æ”¶æ•›æ ‡å‡†æ˜¯ç›¸é‚»ä¸¤æ¬¡æ¯”è¾ƒçš„å…¨å¸§ä¸Žé�žç©ºå¸§å�˜åŒ–çŽ‡å�Šå…¶åŒºé—´ä¸Šç•Œå�‡ä½ŽäºŽ1%ï¼›è¿™æ˜¯å€™é€‰ä¸€è‡´æ€§æ ‡å‡†ï¼Œç‹¬ç«‹äºŽåŽŸ49å¸§äººå·¥/ä»£ç�†å®¡è®¡å�œæ­¢æ�¡ä»¶ã€‚u   ## å…±å�Œé˜ˆå€¼ä¸‹çš„å·®å¼‚uƒ   | æ¨¡åž‹å¯¹ | å®žè´¨å�˜åŒ–å¸§ | é�žç©ºå¸§å�˜åŒ–çŽ‡ | æ¡†æ•°é‡�æ”¹å�˜å¸§ | æ–°å¢ž/æ¶ˆå¤±æ¡† | åŒ¹é…�æ¡†å¹³å�‡IoU | å�Œæ–¹ç©ºæ¡†å¸§ |z#|---|---:|---:|---:|---:|---:|---:|r<  ry  r  rN  u   â€”z.4fz| Úfromu    â†’ Útoz | r,  ú/rX  z (rZ  z) | r[  r(  r#  r$  r%  z |)r”  u|   | æ¨¡åž‹å¯¹ | å®žè´¨å�˜åŒ–çŽ‡95%ç»„çº§åŒºé—´ | IoU=0.90è§„åˆ™å�˜åŒ–çŽ‡ | åŒ¹é…�æ¡†æœ€å¤§è¾¹ç•Œå��ç§»å�‡å€¼/p95ï¼ˆåƒ�ç´ ï¼‰ |z|---|---:|---:|---:|ra  r_  u   â€“c              3   ó4   K  — | ]}t          |¦  «        V — Œd S r   )r’  r  s     r   r�   z"report_markdown.<locals>.<genexpr>w  s(   è è € ÐLÐL¨a�w q™zœzÐLÐLÐLÐLÐLÐLr!   r.  rQ  r\  )r”  u	   ## åˆ¤æ–­r”  r�  u½   ç¬¬ä¸‰è½®è®­ç»ƒã€�å…¨é‡�é¢„æµ‹å�Šä¿®æ­£åŒ…å®¡è®¡å°šæœªå®Œæˆ�ï¼Œä¸�èƒ½ç»™å‡ºä¸‰è½®ç¨³å®šç»“è®ºã€‚å½“å‰�ç»“æžœå�¯è¯†åˆ«ç¬¬ä¸€è‡³ç¬¬äºŒè½®çš„å·®å¼‚ï¼Œä½†ä¸�èƒ½è¯´æ˜Žæœ€è¿‘ä¸€è½®å·²ç»�æ”¶æ•›ã€‚r‚  uo   ä¸‰è½®åœ¨æœ¬æ¬¡æ“�ä½œæ€§æ ‡å‡†ä¸‹è¾¾åˆ°å€™é€‰ç¨³å®šï¼Œä½†ä¸�è¡¨ç¤ºæœªæ£€æŸ¥æ ‡æ³¨æ­£ç¡®ï¼›ä»�éœ€äººå·¥æ£€æŸ¥ã€‚ul   ä¸‰è½®æœªè¾¾åˆ°æœ¬æ¬¡æ“�ä½œæ€§æ”¶æ•›æ ‡å‡†ï¼Œå‰©ä½™å»ºè®®ä»�éœ€äººå·¥æ£€æŸ¥å’Œä¿®æ­£ï¼›ä¸�èƒ½è‡ªåŠ¨æŽ¥å�—ã€‚é   c                 óF   — g | ]}|d          ¯
|d         d         d         ‘ŒS )r†  r   ry  rZ  rj   r‡  s     r   r  z#report_markdown.<locals>.<listcomp>†  sI   € ð Að Að AØ¨t°JÔ/?ðA�Ð(Ô)¨)Ô4Ð5IÔJð Að Að Ar!   u   é™�ä½Žu   å¢žåŠ æˆ–æŒ�å¹³u   ç›¸é‚»è½®æ¬¡å�˜åŒ–çŽ‡ç”± u    å�˜ä¸º u	   ï¼Œå·®å¼‚uX   ã€‚å�•æ¬¡ä¸‹é™�ä¸�å�•ç‹¬è¯�æ˜Žç¨³å®šï¼›å®Œæ•´åŒºé—´ã€�é…�ç½®åŸŸä¸Žåˆ’åˆ†ç»Ÿè®¡è§�JSONã€‚)	r”  u   ## äººå·¥æ£€æŸ¥å…¥å�£r”  u  `review_priority.csv` ä¸ºå›ºå®šå‰©ä½™èŒƒå›´çš„é€�å¸§æŽ’åº�ï¼šä¼˜å…ˆæœ€æ–°ç›¸é‚»è½®æ¬¡å¢žåˆ æ¡†ï¼Œå…¶æ¬¡è¾¹ç•Œå�˜åŒ–ï¼›å�Œæ—¶ä¿�ç•™æ—©æœŸå·®å¼‚ã€�ä¸‰è½®å…¨ç©ºæ ‡è®°ã€�åŽŸç”Ÿæ–‡ä»¶å��å’Œå�„è½®æ¡†æ•°ã€‚å…¨ç©ºå¸§ä¹Ÿå¿…é¡»æ£€æŸ¥æ˜¯å�¦æ¼�æ ‡ã€‚æŽ’åº�ä¸�ä¼šè‡ªåŠ¨ä¿®æ”¹AnyLabelingæ ‡ç­¾ã€‚r”  u¦   å…¨é‡�JSONåŒ…å�«å�„åˆ’åˆ†/é…�ç½®åŸŸç»Ÿè®¡ã€�ç»„çº§åŒºé—´ã€�æ¡†IoUã€�åƒ�ç´ /å½’ä¸€åŒ–è¾¹ç•Œå��ç§»ã€�ç½®ä¿¡åº¦å�˜åŒ–å’Œæ¨¡åž‹/é˜ˆå€¼/æ ‡ç­¾/æ¸…å�•/é€�å¸§å»ºè®®å“ˆå¸Œã€‚r”  uj   æœ¬æ¯”è¾ƒæ²¡æœ‰ä½¿ç”¨val/testçœŸå€¼ï¼Œæ²¡æœ‰æŠ¥å‘ŠAPæˆ–æµ‹è¯•æ€§èƒ½ï¼ŒåŽŸå†»ç»“ç ”ç©¶å�‚è€ƒä¿�æŒ�åŽŸæ ·ã€‚r”  r?   )r–   r’  Újoinr\   )rp  Úlinesrˆ  ÚmrN  rŽ  r�  Úintervalr=  Ú	edge_textÚdecisionrú   rû   Útrends                 r   Úreport_markdownr¤  W  s
  € ØDÀbð P°6¸'´?ÀjÒ3PÐ3PÐ/Ð/ð  WIð  Pð  Pð  Pð  RTðB˜V HÔ-Ð.@ÔAÐYð Bð BØ�xÔ Ð!9Ô:ÐlðBð Bð Bð DFðkØlnØ$ bðnàoqð|¨6Ð2EÔ+FÐuð |ð |ð |Ø}ðaàbdØ*¨Bð SØ2ð%4€Eð& �w”ð mð mˆØÐ$Ô% iÔ0ˆØ˜&œ	 &Ô)Ð1ˆuˆu¸!¸F¼)ÀFÔ:KÐ7QÐ7QˆØ�Šð l˜$˜vœ,ð lð l¨T°$¬Zð lð l¸A¸i¼Lð lð lÈ1ÈXÌ;ð lð lÝ  Ð#7Ô!8Ñ9Ô9ðlð lÝ?FÀqÐIfÔGgÑ?hÔ?hðlð là˜OÔ,ðlð là12°7´ðlð là>?À	¼lðlð làOSðlð làXYÐZfÔXgðlð lð lñ 	mô 	mð 	mð 	mð 
ð &ð &ð &ñ &€Eà�w”ð Zð ZˆØÐ*Ô+¨IÔ6ˆØÐ+Ô,ˆØ!# K¤ð M�E�EØ—J’JÐLÐL°2Ð6JÔ3KÐLÑLÔLÑLÔLð 	àÐ+Ô,ˆØ" 6œ]Ð2�E�E¸5À¼=Ð8aÐ8aÐ8aÈuÐUZÌ|Ð8aÐ8aÐ8aˆ	Ø�Šð Y˜$˜vœ,ð Yð Y¨T°$¬Zð Yð Y¸Hð Yð YÝ! 'Ð*DÔ"EÑFÔFðYð YØKTðYð Yð Yñ 	Zô 	Zð 	Zð 	Zà	Ð"Ð"Ð"Ñ"€EØ�kÔ"€HØˆg„˜*Ò$Ð$Ø�Šð uñ 	vô 	vð 	vð 	và	�(Ô	ð EØ�Šð  Gñ  	Hô  	Hð  	Hð  	Hà�Šð  Dñ  	Eô  	Eð  	EÝ
ˆ6�'Œ?ÑÔ˜qÒ Ð ðAð AØ" 7œOðAñ Aô A‰ˆˆ1à šE˜E��Ð'8ˆØ�Šð mµ¸±´ð mð mÅWÈQÁZÄZð mð mÐZ_ð mð mð mñ 	nô 	nð 	nà	ð @ð @ð @ñ @€Eð �9Š9�UÑÔÐr!   c                 ó@	  ‡ ‡‡‡‡ — g g cŠ }|D ]G\  }}}t          ||||||¦  «        \  }	}
‰                      |	¦  «         |                     |
¦  «         ŒH‰ j        dz  }|                     ¦   «         r"t	          t          |¦  «        |k    d¦  «         nt          ||¦  «         t          d„ |D ¦   «         ¦  «        Šg g c}Št          j	        t          t          |¦  «        ¦  «        d¦  «        D ] \  }}t          ‰ |         ‰ |         |‰¦  «        \  }}t          ‰ |         ‰ |         |d ¦  «        \  }}|                     ||         d         ||         d         ||dz   k    ||dœ¦  «         ‰                     d„ |D ¦   «         ¦  «         Œ¡t          |¦  «        ‰ j        k    }|rd	nd
t          j        t!          d¦  «        ¦  «                             ¦   «         ||‰|t%          ||¦  «        ddœ}|r<t'          ˆˆ fd„|D ¦   «         ¦  «        t'          ˆfd„|D ¦   «         ¦  «        ddœ|d<   t          ‰ j        dz  |¦  «         ‰ j        dz                       t+          |¦  «        ¦  «         d„ t-          |¦  «        D ¦   «         }‰|d                  }g }|                     ¦   «         D ]ñ\  Š}
|‰         }|                     ‰|
d         |
d         |
d         |
d         |
d         |d         |d         |d         |d         t1          t3          ˆfd„‰D ¦   «         ¦  «        ¦  «        t1          t5          ˆˆfd „‰ D ¦   «         ¦  «        ¦  «        d!                     ˆˆfd"„‰ D ¦   «         ¦  «        |d#         rt9          |d#         ¦  «        nd$d%œ¦  «         Œò|                     d&„ ¬'¦  «         t-          |d¦  «        D ]
\  }}||d(<   Œ‰ j        d)z                       d*d$d+¬,¦  «        5 }t?          j         |d(gd-„ |d         D ¦   «         z   ¬.¦  «        }| !                    ¦   «          | "                    |¦  «         d d d ¦  «         n# 1 swxY w Y   tG          |‰ j        ¦  «         t          ‰ j        d/z  ˆ fd0„d1D ¦   «         ¦  «         |S )2Nzprediction_inputs.jsonz;Saved prediction inputs changed; use a new analysis versionc              3   ó&   K  — | ]}|d          V — ŒdS )rß   Nrj   rÓ   s     r   r�   zanalysis.<locals>.<genexpr>   s(   è è € Ð;Ð;¨1�Ð%Ô&Ð;Ð;Ð;Ð;Ð;Ð;r!   r<   r   r¼   )r—  r˜  r†  Únative_confidencer   c                 ó    — i | ]}|d          |“ŒS )rh   rj   r…   s     r   rn   zanalysis.<locals>.<dictcomp>§  s   € ÐGÐGÐG°S˜C 
œO¨SÐGÐGÐGr!   rÎ   ÚpartialúAsia/ShanghaiF)rÍ   Ú
created_atrx   r‡   r   r<  r�  Útest_ground_truth_accessedc              3   óR   •‡K  — | ] Št          ˆˆfd „‰D ¦   «         ¦  «        V — Œ!dS )c              3   ó†   •K  — | ];}|‰         d           p't          ˆfd„|‰         d          D ¦   «         ¦  «         V — Œ<dS )rµ   c              3   ó0   •K  — | ]}|d          ‰k    V — ŒdS ©r»   Nrj   ©rk   rû   Úcommons     €r   r�   z/analysis.<locals>.<genexpr>.<genexpr>.<genexpr>°  s,   øè è € Ð/iÐ/iÈ°°'´
¸fÒ0DÐ/iÐ/iÐ/iÐ/iÐ/iÐ/ir!   N©Úany©rk   Úrr²  rJ   s     €€r   r�   z%analysis.<locals>.<genexpr>.<genexpr>¯  sx   øè è € ð (9ð (9à,-ð -.¨c¬FÐ3EÔ,FÐ(Fð )jÝ,/Ð/iÐ/iÐ/iÐ/iÈaÐPSÌfÐUgÔNhÐ/iÑ/iÔ/iÑ,iÔ,iÐ(ið(9ð (9ð (9ð (9ð (9ð (9r!   N©rÀ   )rk   rJ   r²  Úloadeds    @€€r   r�   zanalysis.<locals>.<genexpr>¯  su   øøè è € ð $Oð $Oà=@õ %(ð (9ð (9ð (9ð (9ð (9à17ð(9ñ (9ô (9ñ %9ô %9ð $Oð $Oð $Oð $Oð $Oð $Or!   c              3   óP   •‡K  — | ]Št          ˆfd „‰D ¦   «         ¦  «        V — Œ dS )c              3   ó6   •K  — | ]}|‰         d           V — ŒdS rd  rj   ©rk   r_   rJ   s     €r   r�   z%analysis.<locals>.<genexpr>.<genexpr>²  s0   øè è € Ð<hÐ<hÐWXÀÀ3ÄÈ	ÔARÐ=RÐ<hÐ<hÐ<hÐ<hÐ<hÐ<hr!   Nr·  )rk   rJ   Úframe_pairss    @€r   r�   zanalysis.<locals>.<genexpr>²  s^   øøè è € ð 9Nð 9NØ<?õ :=Ð<hÐ<hÐ<hÐ<hÐ\gÐ<hÑ<hÔ<hÑ9hÔ9hð 9Nð 9Nð 9Nð 9Nð 9Nð 9Nr!   )Úall_empty_framesÚ%no_substantive_pairwise_change_framesr‹  Úthree_round_consensusúcomparison.jsonúreport_zh.mdc                 ó(   — g | ]\  }}|d          ¯|‘ŒS r…  rj   )rk   r  r_   s      r   r  zanalysis.<locals>.<listcomp>·  s%   € ÐHÐHÐH™d˜a ¸!¸J¼-ÐH˜ÐHÐHÐHr!   ró   r„   r¬   r«   ÚimageÚ
annotationr#  r$  r*  r,  c              3   ó4   •K  — | ]}|‰         d          V — ŒdS rd  rj   r»  s     €r   r�   zanalysis.<locals>.<genexpr>Á  s-   øè è € Ð?gÐ?gÐVWÀÀ#ÄÀyÔ@QÐ?gÐ?gÐ?gÐ?gÐ?gÐ?gr!   c              3   óh   •K  — | ],}t          ˆfd „|‰         d         D ¦   «         ¦  «         V — Œ-dS )c              3   ó0   •K  — | ]}|d          ‰k    V — ŒdS r°  rj   r±  s     €r   r�   z%analysis.<locals>.<genexpr>.<genexpr>Â  s-   øè è € Ð<vÐ<vÐVW¸Q¸w¼ZÈ6Ò=QÐ<vÐ<vÐ<vÐ<vÐ<vÐ<vr!   rµ   Nr³  rµ  s     €€r   r�   zanalysis.<locals>.<genexpr>Â  sl   øè è € ð 5Fð 5FØ9:õ :=Ð<vÐ<vÐ<vÐ<vÐ[\Ð]`Ô[aÐbtÔ[uÐ<vÑ<vÔ<vÑ9vÔ9vÐ5vð 5Fð 5Fð 5Fð 5Fð 5Fð 5Fr!   ú|c              3   ó€   •K  — | ]8}t          t          ˆfd „|‰         d         D ¦   «         ¦  «        ¦  «        V — Œ9dS )c              3   ó0   •K  — | ]}|d          ‰k    V — ŒdS r°  rj   r±  s     €r   r�   z%analysis.<locals>.<genexpr>.<genexpr>Ä  s-   øè è € Ð=wÐ=wÐWX¸aÀ¼jÈFÒ>RÐ=wÐ=wÐ=wÐ=wÐ=wÐ=wr!   rµ   N)r—   r8  rµ  s     €€r   r�   zanalysis.<locals>.<genexpr>Ä  sq   øè è € ð 6Fð 6FØ9:õ 7:½#Ð=wÐ=wÐ=wÐ=wÐ\]Ð^aÔ\bÐcuÔ\vÐ=wÑ=wÔ=wÑ:wÔ:wÑ6xÔ6xð 6Fð 6Fð 6Fð 6Fð 6Fð 6Fr!   r  r”  )rh   r„   r¬   r«   rÃ  rÄ  Úlatest_addedÚlatest_deletedÚlatest_geometry_changedÚlatest_substantive_changeÚany_pair_substantive_changeÚall_rounds_emptyÚround_box_countsÚmin_latest_matched_iouc                 óf   — | d         | d         z    | d          | d          | d          | d         fS )NrË  rÌ  rÎ  rÍ  rÏ  rh   rj   )r¶  s    r   r&   zanalysis.<locals>.<lambda>Ç  sH   €  1 ^Ô#4°qÐ9IÔ7JÑ#JÐ!KØ"#Ð$?Ô"@Ð!@À1ÐE^ÔC_ÐB_Ø"#Ð$AÔ"BÐ!BÀAÀjÄMð!S€ r!   )rÂ   Úpriority_rankúreview_priority.csvÚwz	utf-8-sig)ÚnewlineÚencodingc                 ó   — g | ]
}|d k    ¯|‘ŒS )rÔ  rj   )rk   Úks     r   r  zanalysis.<locals>.<listcomp>Í  s$   € ÐGwÐGwÐGwÈaÐbcÐgvÒbvÐbvÈÐbvÐbvÐbvr!   )Ú
fieldnameszanalysis_artifacts.jsonc                 ó@   •— i | ]}|t          ‰j        |z  ¦  «        “ŒS rj   )r/   r{   )rk   rc   rž   s     €r   rn   zanalysis.<locals>.<dictcomp>Ñ  s=   ø€ ð 4cð 4cð 4cÐY]°D½&ÀÄÈtÑASÑ:TÔ:Tð 4cð 4cð 4cr!   )rÀ  rÁ  rÕ  úproposal_change_rates.png)$rï   r–   r{   r�   rP   r    rG   r4  Ú	itertoolsÚcombinationsÚranger\   r  rˆ   r   Únowr   Ú	isoformatr�  r8  rD   r¤  Ú	enumerater]   r  r´  rÀ   r�  r  Úsortr)   ÚcsvÚ
DictWriterÚwriteheaderÚ	writerowsÚ	make_plot)!rž   r‡   ru   rw   r¦   rî   rc   r   r¥   rë   rt   Úinputs_pathr<  r@  rA  ÚsharedÚ
frame_rowsÚnativer€   rÎ   rp  Úadjacent_indicesÚcurrent_pairÚpriorityÚcurrentÚrankrl   r%   Úwriterr²  rJ   r¼  r¸  s!   `                            @@@@r   Úanalysisrô  •  s<  øøøøø€ Ø˜RÐ€FˆJØ!ð "ð "‰ˆˆd�CÝ! $¨¨c°5¸)ÀTÑJÔJ‰ˆˆfØ�Š�dÑÔÐØ×Ò˜&Ñ!Ô!Ð!Ð!Ø”+Ð 8Ñ8€KØ×ÒÑÔð 'Ý•�[Ñ!Ô! ZÒ/Ð1nÑoÔoÐoÐoåˆk˜:Ñ&Ô&Ð&ÝÐ;Ð;°
Ð;Ñ;Ô;Ñ;Ô;€FØ˜RÐ€Eˆ;ÝÔ(­­s°6©{¬{Ñ);Ô);¸QÑ?Ô?ð Ið I‰ˆˆBÝ(¨°¬°V¸B´ZÀÈFÑSÔSÑˆ�
Ý  r¤
¨F°2¬J¸	À4ÑHÔH‰	ˆ�Ø�Š˜f Rœj¨œm°6¸"´:¸a´=ÈbÐTVÐYZÑTZÊlØ+1ÈðPð Pñ 	Qô 	Qð 	Qà×ÒÐGÐG¸JÐGÑGÔGÑHÔHÐHÐHÝ�6‰{Œ{˜dÔ2Ò2€HØ%-Ð<�z�z°9Ý$œL­°/Ñ)BÔ)BÑCÔC×MÒMÑOÔOØ¨
ÈÐZ_Ý$ U¨HÑ5Ô5ÐUZð\ð \€Fð ð ,å #ð $Oð $Oð $Oð $Oð $OàDMð$Oñ $Oô $Oñ !Oô !Oõ 69ð 9Nð 9Nð 9Nð 9NØCLð9Nñ 9Nô 9Nñ 6Nô 6Nà%*ð+,ð +,ˆÐ&Ñ'õ 
ˆ$Œ+Ð)Ñ
)¨6Ñ2Ô2Ð2Ø	„[�>Ñ!×-Ò-­o¸fÑ.EÔ.EÑFÔFÐFØHÐH¥i°Ñ&6Ô&6ÐHÑHÔHÐØÐ/°Ô3Ô4€LØ€HØ —’Ñ(Ô(ð eð e‰ˆˆVØ˜sÔ#ˆØ�Š S°6¸'´?ÈfÐU]ÔN^Ø%+¨JÔ%7À&ÈÄ/Ø'-¨lÔ';ÈWÐU\ÔM]Ø+2°9Ô+=ÐZaÐbzÔZ{Ø6=¸iÔ6HÝ8;½CÐ?gÐ?gÐ?gÐ?gÐ[fÐ?gÑ?gÔ?gÑ<gÔ<gÑ8hÔ8hÝ-0µð 5Fð 5Fð 5Fð 5Fð 5FØ>Dð5Fñ 5Fô 5Fñ 2Fô 2Fñ .Gô .Gà-0¯XªXð 6Fð 6Fð 6Fð 6Fð 6FØ>Dð6Fñ 6Fô 6Fñ .Fô .FàKRÐSYÌ?Ð3bµ3°w¸v´Ñ3GÔ3GÐ3GÐ`bð
dð 
dñ 
	eô 
	eð 
	eð 
	eð ‡M‚Mð Sð S€Mñ Tô Tð Tõ ˜x¨Ñ+Ô+ð $ð $‰	ˆˆcØ#ˆˆOÑÐØ
Œ+Ð-Ñ
-×	3Ò	3°CÀÈkÐ	3Ñ	ZÔ	Zð #Ð^dÝ” °OÐ3DÐGwÐGwÐS[Ð\]ÔS^ÐGwÑGwÔGwÑ3wÐxÑxÔxˆØ×ÒÑÔÐØ×Ò˜Ñ"Ô"Ð"ð#ð #ð #ñ #ô #ð #ð #ð #ð #ð #ð #øøøð #ð #ð #ð #õ ˆf�d”kÑ"Ô"Ð"Ý	ˆ$Œ+Ð1Ñ
1ð 4cð 4cð 4cð 4cØ
að4cñ 4cô 4cñ dô dð dà€Ms   Ï:AQÑQÑ!Qc                 ó¸  — dd l }|                     d¦  «         dd lm} g g g }}}| d         D ]}}|                     |d         dz   |d         z   ¦  «         |                     |d         d         d	         d
z  ¦  «         |                     |d         d         d	         d
z  ¦  «         Œ~t          j        t          |¦  «        ¦  «        }|                     d¬¦  «        \  }	}
|
 	                    |dz
  |dd¬¦  «         |
 	                    |dz   |dd| d         d›�¬¦  «         |
 
                    dddd¬¦  «         |
                     ||¦  «         |
                     d¦  «         |
                     d¦  «         |
                     d¬¦  «         |	                     ¦   «          |	                     |dz  d¬¦  «         |                     |	¦  «         d S ) Nr   ÚAggr<  r—  u   
â†’ r˜  r§  ry  rZ  éd   r   )é   g      @)Úfigsizeg
×£p=
Ç?g
×£p=
×?zDelivered confidence thresholds)ÚlabelzCommon threshold r–  r¼   Úredz--zOperational 1% criterion)ÚcolorÚ	linestylerú  z,Frames with substantive proposal changes (%)z8Same unreviewed cohort; proposal stability, not accuracyrø  )ÚfontsizerÝ  é    )Údpi)Ú
matplotlibÚuseÚmatplotlib.pyplotÚpyplotr–   r¾   Úaranger\   ÚsubplotsÚbarÚaxhlineÚ
set_xticksÚ
set_ylabelÚ	set_titleÚlegendÚtight_layoutÚsavefigÚclose)rp  r{   r  ÚpltÚlabelsrí  r²  rˆ  ÚxÚfigÚaxs              r   ré  ré  Ö  sò  € ØÐÐÐØ‡N‚N�5ÑÔÐØ#Ð#Ð#Ð#Ð#Ð#Ø  R�FˆF€FØ�w”ð Xð XˆØ�Š�d˜6”l XÑ-°°T´
Ñ:Ñ;Ô;Ð;Ø�Š�dÐ.Ô/°	Ô:Ð;OÔPÐSVÑVÑWÔWÐWØ�Š�dÐ.Ô/°	Ô:Ð;OÔPÐSVÑVÑWÔWÐWÐWÝ
Œ	•#�f‘+”+ÑÔ€AØ�lŠl 8ˆlÑ,Ô,�G€CˆØ‡F‚Fˆ1ˆs‰7�F˜CÐ'H€FÑIÔIÐIØ‡F‚Fˆ1ˆs‰7�F˜CÐ'\¸6ÐBUÔ;VÐ'\Ð'\Ð'\€FÑ]Ô]Ð]Ø‡J‚Jˆq˜¨Ð5O€JÑPÔPÐPØ‡M‚M�!�VÑÔÐØ‡M‚MÐ@ÑAÔAÐAØ‡L‚LÐKÑLÔLÐLØ‡I‚I�q€IÑÔÐØ×ÒÑÔÐØ‡K‚K�Ð4Ñ4¸#€KÑ>Ô>Ð>Ø‡I‚Iˆc�N„N€N€N€Nr!   c                 óð   — 	 t          j        | d¦  «         t          d| › d�¦  «                             ¦   «                              dd¦  «        d         d         dk    S # t
          t          f$ r Y dS w xY w)Nr   z/proc/z/statz) r¼   ÚZF)ÚosÚkillr   r   r„   ÚFileNotFoundErrorÚProcessLookupError)Úpids    r   Ú	pid_aliver  í  sƒ   € ðÝ
Œ��Q‰ŒˆÝÐ'˜SÐ'Ð'Ð'Ñ(Ô(×2Ò2Ñ4Ô4×:Ò:¸4ÀÑCÔCÀAÔFÀqÔIÈSÒPÐPøÝÕ1Ð2ð ð ð Øˆuˆuðøøøs   ‚AA  Á A5Á4A5c                 ó˜  — t          | j        d                              d¦  «        d         ¦  «                             ¦   «         }t	          |dz  ¦  «        }|dt          |¦  «        › d�z  }t          |d         dk    d	¦  «         t          |d         d
k    o0|d         d
k    o$|d         t          |¦  «        k    o|d         dk    d¦  «         t          |d         t          t          d„ | 	                    ¦   «         D ¦   «         ¦  «        ¦  «        k    d¦  «         t          t          |¦  «        |d         k    d¦  «         t          |                     d¦  «        du d¦  «         dt          |¦  «        |d         d         t          |¦  «        |d         t          | j        dz                       ¦   «         ¦  «        t          | j        dz                       ¦   «         ¦  «        t          | j        dz  ¦  «        |d         dddœ}t          | j        d z  |¦  «         t          t          ¦  «                             ¦   «         j        j        }d!| j        j        z   d"z   }d#|› d$t%          j        t)          d%¦  «        ¦  «        d&›d'|d         › d(t          |¦  «        › d)|                     |¦  «        › d*|d         › d+|d         d,         rd-nd.› d/| j        dz                       ¦   «                              |¦  «        › d0�}	d1D ]7}
||
z  }|                     ¦   «         }||vr|                     ||	z   ¦  «         Œ8|S )2Nró   rz   r<   úpackage_audit.jsonÚhuman_correction_remaining_z_v1.ziprÍ   rÎ   z!Three-round comparison incompleteÚpassedÚzip_crcrX  Úmanual_overlapr   z5Correction package failed audit or has wrong coveragerŒ   c              3   ó&   K  — | ]}|d          V — ŒdS rƒ   rj   r…   s     r   r�   z!final_delivery.<locals>.<genexpr>ý  s&   è è € Ð1]Ð1]À3°#°g´,Ð1]Ð1]Ð1]Ð1]Ð1]Ð1]r!   z)Correction package split coverage changedÚ
zip_sha256zCorrection archive changedÚvalidation_test_labels_scoredFz$Package used independent test labelsrx   rŠ   rÁ  rÕ  rÀ  r�  T)rÍ   r‹   rŠ   Úcorrection_zipÚcorrection_zip_sha256Ústability_reportÚpriority_csvÚcomparison_sha256Ústability_judgementÚanalysis_does_not_modify_labelsr¬  zdelivery_manifest.jsonz<!-- manual-proposal-stability:z -->z

z
## rª  z%Y-%m-%du;   ï¼šå‰©ä½™ä¿®æ­£åŒ…å�Šä¸‰è½®å»ºè®®ç¨³å®šæ€§

å·²æŽ’é™¤ç´¯è®¡u   äººå·¥æ£€æŸ¥å¸§ï¼Œå‰©ä½™u(   å¸§ä¿®æ­£åŒ…å…¨é‡�å®¡è®¡é€šè¿‡ã€‚ZIPï¼š`u   `ï¼›SHA-256 `uI   `ã€‚å›ºå®šå�Œä¸€æœªæ£€æŸ¥èŒƒå›´å®Œæˆ�ä¸‰è½®å»ºè®®å·®å¼‚æ¯”è¾ƒï¼Œç»“è®ºï¼šr‚  u!   è¾¾åˆ°æ“�ä½œæ€§å€™é€‰ç¨³å®šæ ‡å‡†u$   æœªè¾¾åˆ°æ“�ä½œæ€§å€™é€‰æ”¶æ•›æ ‡å‡†u=   ã€‚ç¨³å®šæ€§ä¸�ä»£è¡¨æ­£ç¡®ï¼Œä»�éœ€äººå·¥æ£€æŸ¥ã€‚æŠ¥å‘Šï¼š`u)   `ï¼›ä¼˜å…ˆæ£€æŸ¥CSVä¸ŽæŠ¥å‘Šå�Œç›®å½•ã€‚
)zdocs/training_log.mdzdocs/change_log.mdzdocs/session_summary.mdz	README.md)r   r”   re   r•   r    r\   rP   r™   r   rš   r/   r½   r—   r{   rG   rœ   r@   rc   r   rá  r   Úrelative_tor   rD   )rž   rp  rw   ÚlatestÚauditÚarchiveÚdeliveryÚprojectÚmarkerÚtextÚfilenamer   rñ  s                r   Úfinal_deliveryr6  õ  sœ  € Ý�$”*˜R”.×*Ò*¨3Ñ/Ô/°Ô2Ñ3Ô3×;Ò;Ñ=Ô=€FÝ�Ð.Ñ.Ñ/Ô/€EØÐLµS¸±^´^ÐLÐLÐLÑL€GÝˆF�7ŒO˜zÒ)Ð+NÑOÔOÐOÝˆE�'ŒN˜hÒ&ð S¨5°Ô+;¸xÒ+Gð SØ�h”¥3 y¡>¤>Ò1ðSØ6;Ð<LÔ6MÐQRÒ6RØCñEô Eð Eõ ˆE�.Ô!¥T­'Ð1]Ð1]È)×JZÒJZÑJ\ÔJ\Ð1]Ñ1]Ô1]Ñ*]Ô*]Ñ%^Ô%^Ò^Ø7ñ9ô 9ð 9å�F�7‰OŒO˜u \Ô2Ò2Ð4PÑQÔQÐQÝˆE�IŠIÐ5Ñ6Ô6¸%Ð?ÐAgÑhÔhÐhØ#½¸Y¹¼Ø*0°Ô*:Ð;SÔ*TÝ"% g¡,¤,ÈÈ|ÔI\Ý$'¨¬°~Ñ)E×(NÒ(NÑ(PÔ(PÑ$QÔ$QÝ # T¤[Ð3HÑ%H×$QÒ$QÑ$SÔ$SÑ TÔ TÝ%+¨D¬KÐ:KÑ,KÑ%LÔ%LØ'-¨kÔ':Ø37ÐW\ð^ð ^€Hõ 
ˆ$Œ+Ð0Ñ
0°(Ñ;Ô;Ð;Ý•8‰nŒn×$Ò$Ñ&Ô&Ô-Ô4€GØ.°´Ô1AÑAÀFÑJ€FðC�6ð Cð C¥¤­X°oÑ-FÔ-FÑ GÔ Gð  Að Cð CØ&Ð'?Ô@ðCð CÝZ]Ð^gÑZhÔZhðCð Cà×)Ò)¨'Ñ2Ô2ðCð CàAFÀ|ÔATðCð Cð 7=¸[Ô6IÈ(Ô6SÐÐ2Ð2ÐYð	Cð Cð œ+¨Ñ6×?Ò?ÑAÔA×MÒMÈgÑVÔVðCð Cð C€Dð kð ,ð ,ˆØ˜Ñ!ˆØ—.’.Ñ"Ô"ˆØ˜Ð Ð Ø�OŠO˜G d™NÑ+Ô+Ð+øØ€Or!   c            
      óÂ  — t          j        t          ¬¦  «        } |                      dddd¬¦  «         |                      dt          d¬¦  «         |                      d	t          t	          d
¦  «        ¬¦  «         |                      dt          t	          d¦  «        ¬¦  «         |                      dt          d¬¦  «         |                      dt
          d¬¦  «         |                      ddd¬¦  «         |                      dt
          d¬¦  «         |                      ¦   «         }t          |j        dk    ot          |j
        ¦  «        dv d¦  «         t          |j         pt          |j
        ¦  «        dk    d¦  «         t          |¦  «        \  }}}}|j        �rŒ	 g }|D ]Z\  }}}	|dz  }
|
                     ¦   «         r&t          |
¦  «                             d¦  «        dk    r|                     |¦  «         Œ[|d         d         }|s|d z                       ¦   «         rnðd!||d z                       ¦   «          t          |¦  «        |d"         t#          j        t'          d#¦  «        ¦  «                             ¦   «         d$œ}|j        rXt-          |j        ¦  «        sD|                     d%d&¬'¦  «         t1          |j        d(z  |¦  «         t5          |d)         ¦  «        ‚t1          |j        d(z  |¦  «         t7          j        d*¦  «         �Œwt          |¦  «        \  }}}}t1          |j        d(z  d+t          |¦  «        d,œ¦  «         	 t;          |||||¦  «        }|j        rt=          |||¦  «         t1          |j        d(z  |d         |d-         t#          j        t'          d#¦  «        ¦  «                             ¦   «         d.œ¦  «         t?          tA          j!        |d         |d-         tE          |j        d/z  ¦  «        d0œd1¬2¦  «        d¬3¦  «         d S # tF          $ r.}t1          |j        d(z  d%tE          |¦  «        d4œ¦  «         ‚ d }~ww xY w)5N)Údescriptionz--roundr–   TzNAME=OUTPUT, oldest first)ÚactionÚrequiredÚhelpz--manual-output)Útyper:  z--source-exportz&outputs/anylabeling/pallet_v5_20260930)r<  Údefaultz
--protocolzoutputs/protocol_v5z--outputz--expected-roundsr›  z--waitÚ
store_truez5Wait for all prediction and correction-package audits)r9  r;  z--workflow-pidz-Exit resumably if the upstream workflow stops)r<  r;  )r<   r›  z&This task compares two or three roundsz --wait requires all three roundsrÌ   rÍ   rÎ   ró   r¼   r  Úwaiting_for_upstreamrŠ   rª  )rÍ   Úpending_roundsÚwaiting_for_package_auditr‹   rŠ   Ú
updated_atÚfailed_resumablez'Upstream stopped before audited package)rÍ   r`  zstatus.jsonr`  é   Ú	analyzing)rÍ   r‹   r�  )rÍ   r�  rB  rÁ  )rÍ   r�  ÚreportF)r2   )Úflush)rÍ   Úerror)$ÚargparseÚArgumentParserÚ__doc__Úadd_argumentr   r  Ú
parse_argsrP   rˆ   r\   r”   Úwaitr¨   r�   r    r½   r–   r   rá  r   râ  Úworkflow_pidr  r+   rG   r{   Ú
SystemExitÚtimeÚsleeprô  r6  Úprintr   r3   r—   Ú	Exception)Úparserrž   r‡   ru   rw   r¦   Úpendingrc   r   r€   rå   r.  rÍ   rp  rH  s                  r   ÚmainrW    s—  € ÝÔ$µÐ9Ñ9Ô9€FØ
×Ò˜	¨(¸TÐHcÐÑdÔdÐdØ
×ÒÐ)µ¸tÐÑDÔDÐDØ
×ÒÐ)µ½dÐCkÑ>lÔ>lÐÑmÔmÐmØ
×Ò˜­4½Ð>SÑ9TÔ9TÐÑUÔUÐUØ
×Ò˜
­¸ÐÑ=Ô=Ð=Ø
×ÒÐ+µ#¸qÐÑAÔAÐAØ
×Ò˜¨Ð<sÐÑtÔtÐtØ
×ÒÐ(­sÐ9hÐÑiÔiÐiØ×ÒÑÔ€DÝˆDÔ  AÒ%ÐC­#¨d¬j©/¬/¸VÐ*CÐEmÑnÔnÐnÝ�”	ˆMÐ1�S ¤™_œ_°Ò1Ð3UÑVÔVÐVÝ%,¨T¡]¤]Ñ"€FˆE�9˜dØ„yñ 7ð	ØˆGØ!'ð )ð )‘��d˜AØÐ"<Ñ<�Ø—’Ñ(Ô(ð )­D°©N¬N×,>Ò,>¸wÑ,GÔ,GÈ:Ò,UÐ,UØ—N’N 4Ñ(Ô(Ð(øØ˜B”Z ”]ˆFØð  Ð)=Ñ =×EÒEÑGÔGð ØØ4ÈØ7=Ð@TÑ7T×6\Ò6\Ñ6^Ô6^Ð2^Ý),¨Y©¬ÐSWÐXpÔSqÝ#+¤<µ¸Ñ0IÔ0IÑ#JÔ#J×#TÒ#TÑ#VÔ#VðXð XˆEð Ô ð 2­°4Ô3DÑ)EÔ)Eð 2Ø—’Ð#5Ð>g�ÑhÔhÐhÝ�d”k MÑ1°5Ñ9Ô9Ð9Ý   x¤Ñ1Ô1Ð1Ý�$”+ Ñ-¨uÑ5Ô5Ð5ÝŒJ�r‰NŒNˆNñ%	õ( *1°©¬Ñ&ˆ��y $Ý	ˆ$Œ+˜Ñ
%°ÕRUÐV_ÑR`ÔR`Ð'aÐ'aÑbÔbÐbð
Ý˜$ ¨¨y¸$Ñ?Ô?ˆØŒ9ð 	4Ý˜4 ¨Ñ3Ô3Ð3ÝˆdŒk˜MÑ)°V¸G´_ÐSYÐZeÔSfÝ$œL­°/Ñ)BÔ)BÑCÔC×MÒMÑOÔOð,Qð ,Qñ 	Rô 	Rð 	Rå�dŒj 6¨'¤?ÀÈÔATÝ$'¨¬°nÑ(DÑ$EÔ$EðGð GØUZð\ñ \ô \Øcgð	iñ 	iô 	ið 	ið 	ið 	iøåð ð ð ÝˆdŒk˜MÑ)Ð5GÕRUÐV[ÑR\ÔR\Ð+]Ð+]Ñ^Ô^Ð^Øøøøøðøøøs   ÍCP& Ð&
QÐ0)QÑQÚ__main__)r   r   )F)2rK  rI  Úcollectionsr   r   rå  r   r'   rÞ  r   r  Úpathlibr   rQ  Úzoneinfor   ÚenvironÚ
setdefaultÚnumpyr¾   Úscipy.optimizer   r˜   r    r/   r7   rG   rK   rP   re   rx   r¨   rÊ   rï   r   r  rG  rK  rS  rw  r  r�  r’  r¤  rô  ré  r  r6  rW  Ú__name__rj   r!   r   ú<module>ra     s÷  ððð ð €€€Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø 
€
€
€
Ø Ð Ð Ð Ð Ð Ø €€€Ø Ð Ð Ð Ø €€€Ø 	€	€	€	Ø Ð Ð Ð Ð Ð Ø €€€Ø Ð Ð Ð Ð Ð ð „
× Ò Ð,¨cÑ 2Ô 2Ð 2Ø „
× Ò Ð'¨Ñ -Ô -Ð -Ø Ð Ð Ð Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0ð QØØØ"Ø!%ØhØsØZð  PØ#Ø$(ð
ð 
€ð.ð .ð .ðð ð ðfð fð fðð ð ð9ð 9ð 9ð"ð "ð "ð
ð ð ð
ð 
ð 
ð%*ð %*ð %*ðP?ð ?ð ?ð,'ð 'ð 'ðTTð Tð Tð_ð _ð _ð _ðJð Jð Jð Jð<<ð <ð <ð4ð 4ð 4ðð ð ð ðBð ð ð Cð Cð Cð*?ð ?ð ?ð;ð ;ð ;ð|>ð >ð >ðBð ð ð.ð ð ð#ð #ð #ðL/ð /ð /ðd ˆzÒÐØ€D�F„F€F€F€Fð Ðr!   