§
    <Æj6  ã            	      óÌ   — d Z ddlmZ ddlmZ ddlZddlZddlZddl	m
Z
 ddlmZmZ ddlmZ d	d
dœd„Zd	d
dœd„Zd„ Zdd„Zd„ Zd„ Zdd„Zd„ Zd„ Zd„ Zdddddddddœd„ZdS )zGCOCO-style box evaluation against independent machine reference labels.é    )Úannotations)Úredirect_stdoutN)Ú
DataLoaderé   )Úcollate_palletÚrestore_boxes)Úmake_loadergü©ñÒMbP?i€  ©ÚconfÚsizec               ó8   — t          | ||g||¬¦  «        d         S )Nr
   r   )Úpredict_batch_boxes)ÚmodelÚimageÚmetar   r   s        úpallet/experiments/evaluate.pyÚpredict_boxesr      s"   € Ý˜u e¨d¨V¸$ÀTÐJÑJÔJÈ1ÔMÐMó    c               óÌ  ‡ ‡‡— dd l }ddlm} t          ‰ d¦  «        rHt	          j        ¦   «         5  ˆˆˆ fd„t          |¦  «        D ¦   «         cd d d ¦  «         S # 1 swxY w Y   t	          j        ¦   «         5   ‰ ‰¦  «        } ||‰ddd¬¦  «        }d d d ¦  «         n# 1 swxY w Y   t          |¦  «        t          |¦  «        k    rt          d	¦  «        ‚g }	t          ||¦  «        D ]v\  }
}|
 
                    ¦   «                              ¦   «         }t          |d d …d d
…f         ||¬¦  «        }|	                     d„ t          ||¦  «        D ¦   «         ¦  «         Œw|	S )Nr   )Únon_max_suppressionÚdecodec                ó\   •— g | ](\  }}‰                      ‰||d z   …         |‰¬¦  «        ‘Œ)S )r   )r   )r   )Ú.0Úindexr   r   Úimagesr   s      €€€r   ú
<listcomp>z'predict_batch_boxes.<locals>.<listcomp>   sL   ø€ ð <ð <ð <Ù#˜˜tð —L’L ¨¨e°a©i¨Ô!8¸$ÀT�LÑJÔJð <ð <ð <r   gffffffæ?r   i,  )Ú
conf_thresÚ	iou_thresÚncÚmax_detz6prediction batch and coordinate metadata counts differé   ©r   c                óf   — g | ].\  }}|                      ¦   «         t          |d          ¦  «        f‘Œ/S )r!   )ÚtolistÚfloat)r   ÚboxÚrows      r   r   z'predict_batch_boxes.<locals>.<listcomp>(   s3   € ÐWÐWÐW¹(¸#¸s˜Ÿš™œ¥u¨S°¬V¡}¤}Ð5ÐWÐWÐWr   )ÚtorchvisionÚultralytics.utils.nmsr   ÚhasattrÚtorchÚinference_modeÚ	enumerateÚlenÚ
ValueErrorÚzipÚcpuÚnumpyr   Úappend)r   r   Úmetadatar   r   r(   r   ÚoutputsÚ
detectionsÚresultsÚ	detectionr   ÚrowsÚboxess   `` `          r   r   r      s*  øøø€ ð ÐÐÐØ9Ð9Ð9Ð9Ð9Ð9åˆu�hÑÔð <ÝÔ!Ñ#Ô#ð 	<ð 	<ð<ð <ð <ð <ð <ð <Ý'0°Ñ':Ô':ð<ñ <ô <ð	<ð 	<ð 	<ð 	<ñ 	<ô 	<ð 	<ð 	<ð 	<ð 	<ð 	<ð 	<øøøð 	<ð 	<ð 	<ð 	<õ 
Ô	Ñ	Ô	ð <ð <Ø�%˜‘-”-ˆØ(Ð(¨¸TÈSØ,-°sð<ñ <ô <ˆ
ð<ð <ð <ñ <ô <ð <ð <ð <ð <ð <ð <øøøð <ð <ð <ð <õ ˆ:�„�#˜h™-œ-Ò'Ð'ÝÐQÑRÔRÐRØ€GÝ˜z¨8Ñ4Ô4ð Yð Y‰ˆ	�4Ø�}Š}‰Œ×$Ò$Ñ&Ô&ˆÝ˜d 1 1 1 b q b 5œk¨4°dÐ;Ñ;Ô;ˆØ�ŠÐWÐWÅcÈ%ÐQUÑFVÔFVÐWÑWÔWÑXÔXÐXÐXØ€Ns#   ±AÁAÁ"AÁ9B!Â!B%Â(B%c                ó
  — t          | d         |j        ¦  «        t          | d         |j        ¦  «        }}t          | d         |j        |j        z   ¦  «        t          | d         |j        |j        z   ¦  «        }}t          d||z
  ¦  «        t          d||z
  ¦  «        z  }t          d| d         | d         z
  ¦  «        t          d| d         | d         z
  ¦  «        z  }||j        |j        z  z   |z
  }|dk    r||z  ndS )Nr   r   é   é   ç        )ÚmaxÚxÚyÚminÚwidthÚheight)	r&   ÚgtÚx1Úy1Úx2Úy2ÚintersectionÚareaÚunions	            r   Ú_iourM   ,   sí   € Ý��Q”˜œÑÔ¥ C¨¤F¨B¬DÑ 1Ô 1ˆ€BÝ��Q”˜œ ¤™Ñ)Ô)­3¨s°1¬v°r´t¸b¼iÑ7GÑ+HÔ+Hˆ€BÝ�s˜B ™GÑ$Ô$¥s¨3°°R±Ñ'8Ô'8Ñ8€LÝˆs�C˜”F˜S œV‘OÑ$Ô$¥s¨3°°A´¸¸Q¼±Ñ'@Ô'@Ñ@€DØ�2”8˜bœiÑ'Ñ'¨,Ñ6€EØ#(¨1¢9 9ˆ<˜%ÑÐ°#Ð5r   © c                ób  ‡
‡— t          ¦   «         Šdx}}t          | d„ ¬¦  «        D ]t\  Š
}||k     rŒˆ
ˆfd„t          |¦  «        D ¦   «         }t          |d„ d¬¦  «        \  }}	|	dk    r‰                     |¦  «         |d	z  }Œ_t          ‰
|¦  «        s|d	z  }Œu||t          |¦  «        |z
  fS )
Nr   c                ó   — | d          S ©Nr   rN   ©Úpairs    r   ú<lambda>z _count_matches.<locals>.<lambda>8   s   € °d¸1´g°X€ r   ©Úkeyc                óB   •— g | ]\  }}|‰v¯	|t          ‰|¦  «        f‘ŒS rN   ©rM   ©r   ÚidxrE   r&   Úuseds      €€r   r   z"_count_matches.<locals>.<listcomp>;   s:   ø€ ð *ð *ð *©w¨s°BØ D˜˜ð �D  b™MœMÐ*Ø(˜˜r   c                ó   — | d         S rQ   rN   )Úitems    r   rT   z _count_matches.<locals>.<lambda>=   s
   € ¸¸Q¼€ r   )éÿÿÿÿr>   ©rV   Údefaultç      à?r   )ÚsetÚsortedr-   r?   ÚaddÚ_inside_ignoredr.   )r:   Úground_truthÚ	thresholdÚignore_regionsÚtpÚfpÚscoreÚ
candidatesrZ   Úoverlapr&   r[   s             @@r   Ú_count_matchesrn   5   sî   øø€ Ý‰5Œ5€DØ€K€BˆÝ˜UÐ(=Ð(=Ð>Ñ>Ô>ð 
?ð 
?‰
ˆˆUØ�9ÒÐØð*ð *ð *ð *ð *½)ÀLÑ:QÔ:Qð *ñ *ô *ˆ
å˜:Ð+?Ð+?ÈÐSÑSÔS‰ˆˆWØ�cŠ>ˆ>Ø�HŠH�S‰MŒMˆMØ�!‰GˆBˆBå" 3 ~Ñ6Ô6Ð>°r¸Q±w°røØˆr•3�|Ñ$Ô$ rÑ)Ð)Ð)r   c           
     ó  — t          d| d         | d         z
  ¦  «        t          d| d         | d         z
  ¦  «        z  }|dk    rdS |D ]¼}t          |d¦  «        r|j        |j        |j        |j        fn|\  }}}}t          dt          | d         ||z   ¦  «        t          | d         |¦  «        z
  ¦  «        t          dt          | d         ||z   ¦  «        t          | d         |¦  «        z
  ¦  «        z  }||z  dk    r dS Œ½dS )	Nr   r<   r=   r   Fr@   ra   T)r?   r*   r@   rA   rC   rD   rB   )	r&   ÚregionsrK   Úregionr@   rA   ÚwÚhrJ   s	            r   re   re   F   s  € Ý	ˆQˆs�1Œv�c˜!”f‰}Ñ	Ô	�c ! C¨¤F¨3¨q¬6¡MÑ2Ô2Ñ	2€DØˆQ‚w€w�e�eØð -ð -ˆÝDKÈFÐSVÑDWÔDWÐc�6”8˜FœH V¤\°&´-Ð@Ð@Ð]c‰ˆˆ!ˆAˆaÝ˜�3˜s 1œv a¨¡c™?œ?­3¨s°1¬v°a©=¬=Ñ8Ñ9Ô9½#¸aÅÀCÈÄFÈ1ÈQÉ3ÁÄÕPSÐTWÐXYÔTZÐ[\ÑP]ÔP]Ñ@]Ñ:^Ô:^Ñ^ˆØ˜Ñ˜bÒ Ð ¨¨¨Ð Øˆ5r   c           
     ód  — t          d„ | D ¦   «         ¦  «        }t          d„ | D ¦   «         ¦  «        }t          d„ | D ¦   «         ¦  «        }||z   r|||z   z  nd }||z   r|||z   z  nd }||z   rd|z  d|z  |z   |z   z  nd }t          | ¦  «        ||z   |||| r|t          | ¦  «        z  nd |||dœ	S )Nc              3  ó&   K  — | ]}|d          V — ŒdS )ri   NrN   ©r   r'   s     r   ú	<genexpr>z_rates.<locals>.<genexpr>Q   ó&   è è € Ð'Ð'˜3ˆS�ŒYÐ'Ð'Ð'Ð'Ð'Ð'r   c              3  ó&   K  — | ]}|d          V — ŒdS )rj   NrN   rv   s     r   rw   z_rates.<locals>.<genexpr>R   rx   r   c              3  ó&   K  — | ]}|d          V — ŒdS )ÚfnNrN   rv   s     r   rw   z_rates.<locals>.<genexpr>S   rx   r   r<   )	r   ÚobjectsÚtrue_positivesÚfalse_positivesÚmissedÚfalse_positives_per_frameÚ	precisionÚrecallÚf1)Úsumr.   )r9   ri   rj   r{   r�   r‚   rƒ   s          r   Ú_ratesr…   P   sø   € Ý	Ð'Ð' $Ð'Ñ'Ô'Ñ	'Ô	'€BÝ	Ð'Ð' $Ð'Ñ'Ô'Ñ	'Ô	'€BÝ	Ð'Ð' $Ð'Ñ'Ô'Ñ	'Ô	'€BØ"$ r¡'Ð3��b˜2‘g‘�¨t€IØ! B™wÐ0ˆR�2˜‘7‰^ˆ^¨D€FØ(*¨R©Ð	9ˆˆR‰�1�r‘6˜B‘; Ñ#Ñ	$Ð	$°T€BÝ˜$‘i”i¨B°©GÀrØ!¨RØ;?Ð)I¨­c°$©i¬i©¨ÀTØ"¨f¸Bð@ð @ð @r   c                ó¬  — t          ||||¬¦  «        }g }|D ]»}|d                              |t          |¦  «                             d¦  «        ¬¦  «        }t	          | ||d         |j        ¬¦  «        }	t          |d         |	¦  «        D ]G\  }
}|j        t          |¦  «                 \  }}d„ |D ¦   «         }| 	                    |||
|f¦  «         ŒHŒ¼|S )N)Ú
batch_sizeÚworkersÚdeviceÚimgÚcuda)Únon_blockingr   r"   c                ód   — g | ]-\  }}|d          |d         k    ¯|d         |d         k    ¯)||f‘Œ.S )r<   r   r=   r   rN   ©r   r&   rk   s      r   r   z_collect.<locals>.<listcomp>e   sH   € ð =ð =ð =¡j c¨5Ø˜Aœ  Q¤š˜¨C°¬F°S¸´VªO¨Oð ˜5�\Ø,;¨O¨Or   )
r	   ÚtoÚstrÚ
startswithr   r   r0   r9   r.   r3   )r   Údatasetr‰   r‡   rˆ   ÚloaderÚ	collectedÚbatchr   Úpredictionsr   r:   ÚrecordÚ
annotations                 r   Ú_collectr™   ]   sý   € Ý˜¨ZÀÐQWÐXÑXÔX€FØ€IØð @ð @ˆØ�u”—’ µc¸&±k´k×6LÒ6LÈVÑ6TÔ6T�ÑUÔUˆÝ)¨%°¸¸v¼ÈWÌ\ÐZÑZÔZˆÝ˜u Vœ}¨kÑ:Ô:ð 	@ð 	@‰KˆD�%Ø!(¤­c°)©n¬nÔ!=ÑˆF�Jð=ð =°Eð =ñ =ô =ˆEà×Ò˜f j°$¸Ð>Ñ?Ô?Ð?Ð?ð		@ð
 Ðr   c           
     ó,  ‡‡— t          d„ | D ¦   «         ¦  «        st          d¦  «        ‚g }t          d„ | D ¦   «         ¦  «        }| D ]Ô\  }}}}t          ¦   «         Št	          |d„ ¬¦  «        D ]¬\  Š}ˆˆfd„t          |j        ¦  «        D ¦   «         }t          |d„ d¬	¦  «        \  }}	|	d
k    }
|
r‰                     |¦  «         |
s t          ‰t          |dd¦  «        ¦  «        rŒ{|                     t          |¦  «        t          |
¦  «        f¦  «         Œ­ŒÕ|sdS |                     d„ ¬¦  «         t          j        d„ |D ¦   «         ¦  «        }t          j        d„ |D ¦   «         ¦  «        }d}t          j        t          j        |d¦  «        ¦  «        }|D ]n}t          t          j        | | d¬¦  «        ¦  «        }|rt          ||dz
           ¦  «        nd}d|z  ||z   z  }t          ||t          |¦  «        f¦  «        }Œo|d         S )Nc              3  ó.   K  — | ]\  }}}}|j         V — Œd S ©N)r:   ©r   Ú_r'   s      r   rw   z$_select_threshold.<locals>.<genexpr>l   s,   è è € Ð7Ð7™\˜Q  Q¨ˆsŒyÐ7Ð7Ð7Ð7Ð7Ð7r   z=threshold calibration requires positive validation referencesc              3  óH   K  — | ]\  }}}}t          |j        ¦  «        V — Œd S rœ   )r.   r:   r�   s      r   rw   z$_select_threshold.<locals>.<genexpr>n   s2   è è € ÐFÐF©y¨q°°Q°q�C ¤	™NœNÐFÐFÐFÐFÐFÐFr   c                ó   — | d          S rQ   rN   rR   s    r   rT   z#_select_threshold.<locals>.<lambda>q   s   € °t¸A´w°h€ r   rU   c                óB   •— g | ]\  }}|‰v¯	|t          ‰|¦  «        f‘ŒS rN   rX   rY   s      €€r   r   z%_select_threshold.<locals>.<listcomp>r   s6   ø€ ÐaÐaÐa©v¨s°2ÐQTÐ\`ÐQ`ÐQ`˜�T # b™\œ\Ð*ÐQ`ÐQ`ÐQ`r   c                ó   — | d         S rQ   rN   rR   s    r   rT   z#_select_threshold.<locals>.<lambda>s   s
   € ¸¸a¼€ r   )r^   r   r_   ra   rh   rN   ç      Ð?c                ó   — | d          S )Nr   rN   rR   s    r   rT   z#_select_threshold.<locals>.<lambda>y   s   €  T¨!¤W H€ r   c                ó   — g | ]
}|d          ‘ŒS ©r   rN   ©r   rS   s     r   r   z%_select_threshold.<locals>.<listcomp>z   s   € Ð4Ð4Ð4 �T˜!”WÐ4Ð4Ð4r   c                ó   — g | ]
}|d          ‘ŒS )r   rN   r§   s     r   r   z%_select_threshold.<locals>.<listcomp>{   s   € Ð1Ð1Ð1˜d�$�q”'Ð1Ð1Ð1r   )g      ð¿r£   Úright)Úsider   r   r<   )Úanyr/   r„   rb   rc   r-   r:   r?   rd   re   Úgetattrr3   r%   ÚintÚsortÚnpÚarrayÚcumsumÚuniqueÚsearchsorted)r”   r6   Útotalrž   r'   r:   rk   rl   Úmatchrm   ÚhitÚscoresri   ÚbestÚ
thresholdsrg   ÚcountÚtruerƒ   r&   r[   s                      @@r   Ú_select_thresholdr¼   k   sQ  øø€ ÝÐ7Ð7¨YÐ7Ñ7Ô7Ñ7Ô7ð ZÝÐXÑYÔYÐYØ€J�ÐFÐF¸IÐFÑFÔFÑFÔF�%Ø"ð 7ð 7‰ˆˆ#ˆa�Ý‰UŒUˆÝ Ð*>Ð*>Ð?Ñ?Ô?ð 	7ð 	7‰IˆC�ØaÐaÐaÐaÐa½À3Ä9Ñ9MÔ9MÐaÑaÔaˆJÝ˜jÐ-@Ð-@ÈÐPÑPÔP‰MˆE�'Ø˜’ˆCØÐ"�4—8’8˜E‘?”?�?ØÐY�?¨3­w°sÐ;KÈBÑ/OÔ/OÑPÔPÐYÐQYØ×Ò�u U™|œ|­C°©H¬HÐ5Ñ6Ô6Ð6Ð6ð	7ð Ð ˜S˜SØ‡O‚OÐ,Ð,€OÑ-Ô-Ð-ÝŒ8Ð4Ð4¨Ð4Ñ4Ô4Ñ5Ô5€FÝ	„yÐ1Ð1 jÐ1Ñ1Ô1Ñ2Ô2€BØ	€DÝŒy�œ 6¨#Ñ.Ô.Ñ/Ô/€JØð -ð -ˆ	Ý•"”/ 6 '¨9¨*¸'ÐBÑBÔBÑCÔCˆØ!&Ð-�S��E˜!‘G”ÑÔÐ¨AˆØˆT‰6�5˜‘;ÑˆÝ��r�% 	Ñ*Ô*Ð+Ñ,Ô,ˆˆØ�Œ7€Nr   c                ó€  — ddl m} ddlm} g }g }g }t	          | d¦  «        D �]b\  }}|                     ||                     dd¦  «        |                     dd¦  «        |                     dt          |¦  «        ¦  «        dœ¦  «         |                     d	g ¦  «        D ]=}|                     t          |¦  «        dz   |d||d
         |d         z  ddœ¦  «         Œ>|                     dg ¦  «        D ]>}|                     t          |¦  «        dz   |d||d
         |d         z  dddœ¦  «         Œ?|                     dg ¦  «        D ]9}	|	d         \  }
}}}|                     |d|
|||
z
  ||z
  g|	d         dœ¦  «         Œ:�Œdt          d„ |D ¦   «         ¦  «        sd d d dœS |sddddœS t          t          j        ¦   «         ¦  «        5   |¦   «         }||dddœgi dœ|_        |                     ¦   «          |                     |¦  «        } |||d¦  «        }|                     ¦   «          |                     ¦   «          |                     ¦   «          d d d ¦  «         n# 1 swxY w Y   t'          |j        d         ¦  «        t'          |j        d
         ¦  «        t'          |j        d         ¦  «        dœS )Nr   ©ÚCOCO©ÚCOCOevalr   rC   rD   Úframe_id©ÚidrC   rD   Ú	file_nameÚreference_boxesr<   r=   ©rÄ   Úimage_idÚcategory_idÚbboxrK   Úiscrowdrh   ©rÄ   rÈ   rÉ   rÊ   rK   rË   Úignorer–   Úxyxyrk   ©rÈ   rÉ   rÊ   rk   c              3  óB   K  — | ]}|                      d ¦  «         V — ŒdS )rË   N)Úgetrv   s     r   rw   z_coco_ap.<locals>.<genexpr>—   s1   è è € Ð7Ð7¨#�3—7’7˜9Ñ%Ô%Ð%Ð7Ð7Ð7Ð7Ð7Ð7r   ©ÚAP50ÚAP75ÚAP50_95r>   Úpallet©rÄ   Úname)r   r   Ú
categoriesÚinforÊ   )Úpycocotools.cocor¿   Úpycocotools.cocoevalrÁ   r-   r3   rÑ   r�   r.   r«   r   ÚioÚStringIOr’   ÚcreateIndexÚloadResÚevaluateÚ
accumulateÚ	summarizer%   Ústats)r9   r¿   rÁ   r   Útruthr–   r   r'   r&   Ú
predictionrF   rG   rH   rI   ÚcocoÚresultÚ	evaluators                    r   Ú_coco_aprê   †   sœ  € Ø%Ð%Ð%Ð%Ð%Ð%Ø-Ð-Ð-Ð-Ð-Ð-Ø€F�Bˆe 2�{Ý˜t AÑ&Ô&ð >ñ >‰	ˆˆcØ�Š˜E¨#¯'ª'°'¸!Ñ*<Ô*<ÀcÇgÂgÈhÐWXÑFYÔFYØ#&§7¢7¨:µc¸%±j´jÑ#AÔ#AðCð Cñ 	Dô 	Dð 	Dà—7’7Ð,¨RÑ0Ô0ð 	=ð 	=ˆCØ�LŠL�s 5™zœz¨!™|°uÈ1ÐTWØ!$ Q¤¨¨A¬¡¸ð<ð <ñ =ô =ð =ð =à—7’7Ð+¨BÑ/Ô/ð 	Hð 	HˆCØ�LŠL�s 5™zœz¨!™|°uÈ1ÐTWØ!$ Q¤¨¨A¬¡¸ÀAðGð Gñ Hô Hð Hð HàŸ'š' -°Ñ3Ô3ð 	>ð 	>ˆJØ" 6Ô*‰KˆBˆr�"�RØ×Ò¨5¸qÈÈBÈrÐRTÉuÐUWÐXZÑUZÐH[Ø(2°7Ô(;ð =ð  =ñ >ô >ð >ð >ñ	>õ Ð7Ð7°Ð7Ñ7Ô7Ñ7Ô7ÐgÀtÐSWÐbfÐ?gÐ?gÐ8gØÐ@ b°¸RÐ@Ð@Ð@Ý	�œ™œÑ	'Ô	'ð Jð JØˆT‰VŒVˆ¨6ÀÐZ[ÐckÐTlÐTlÐSmÐuwÐ!xÐ!x�D”LØ×ÒÑÔÐ $§,¢,¨{Ñ";Ô";˜6ÀhÀhÈtÐTZÐ[aÑFbÔFb¸IØ×ÒÑÔÐ˜Y×1Ò1Ñ3Ô3Ð3°I×4GÒ4GÑ4IÔ4IÐ4IðJð Jð Jñ Jô Jð Jð Jð Jð Jð Jð Jøøøð Jð Jð Jð Jõ ˜œ¨Ô+Ñ,Ô,µE¸)¼/È!Ô:LÑ4MÔ4MÕX]Ð^gÔ^mÐnoÔ^pÑXqÔXqÐrÐrÐrs   ÇBI'É'I+É.I+c                óh  ‡— i Š| D ]1}‰                      |d         g ¦  «                             |¦  «         Œ2‰si S t          j                             |¦  «        }t          ‰¦  «        }d„ dD ¦   «         }t          |¦  «        D ]�}ˆfd„|                     |t          |¦  «        d¬¦  «        D ¦   «         }i t          |¦  «        ¥t          |¦  «        ¥}	|                     ¦   «         D ](\  }
}|	|
         �|                     |	|
         ¦  «         Œ)Œ‘d„ |                     ¦   «         D ¦   «         S )NÚgroup_idc                ó   — i | ]}|g “ŒS rN   rN   )r   rV   s     r   ú
<dictcomp>z_bootstrap.<locals>.<dictcomp>¨   s   € ÐuÐuÐu˜3ˆs�BÐuÐuÐur   )r�   r‚   rƒ   r€   rÓ   rÔ   rÕ   c                ó*   •— g | ]}‰|         D ]}|‘ŒŒS rN   rN   )r   rØ   r'   Úgroupss      €r   r   z_bootstrap.<locals>.<listcomp>ª   sA   ø€ ð )ð )ð )˜Ø! $œ<ð)ð )Øð ð )ð )ð )ð )r   T)r   Úreplacec           	     ó`   — i | ]+\  }}||r!d „ t          j        |ddg¦  «        D ¦   «         nd“Œ,S )c                ó,   — g | ]}t          |¦  «        ‘ŒS rN   )r%   )r   r@   s     r   r   z)_bootstrap.<locals>.<dictcomp>.<listcomp>°   s   € ÐGÐGÐG˜q•%˜‘(”(ÐGÐGÐGr   g      @g     `X@N)r¯   Ú
percentile)r   rV   Úvaluess      r   rî   z_bootstrap.<locals>.<dictcomp>°   sa   € ð Dð Dð DÙ$/ C¨ð Øð ÐGÐG¥B¤M°&¸3À¸+Ñ$FÔ$FÐGÑGÔGÐGØðDð Dð Dr   )Ú
setdefaultr3   r¯   ÚrandomÚdefault_rngÚlistÚrangeÚchoicer.   r…   rê   Úitems)r9   ÚsamplesÚseedr'   ÚrngÚnamesÚmetricsrž   ÚdrawÚratesrV   Úvaluerð   s               @r   Ú
_bootstrapr      sb  ø€ Ø€FØð ;ð ;ˆØ×Ò˜#˜jœ/¨2Ñ.Ô.×5Ò5°cÑ:Ô:Ð:Ð:Øð Øˆ	Ý
Œ)×
Ò
 Ñ
%Ô
%€CÝ�‰LŒL€EØuÐuÐ"tÐuÑuÔu€GÝ�7‰^Œ^ð )ð )ˆð)ð )ð )ð ) §
¢
¨5µs¸5±z´zÈ4 
Ñ PÔ Pð )ñ )ô )ˆà1•6˜$‘<”<Ð1¥(¨4¡.¤.Ð1ˆØ!Ÿ-š-™/œ/ð 	)ð 	)‰JˆC�Ø�SŒzÐ%Ø—’˜U 3œZÑ(Ô(Ð(øð	)ðDð DØ3:·=²=±?´?ðDñ Dô Dð Dr   zcuda:0r£   é*   z*independently reviewed machine annotations)r‰   r‡   rˆ   Úconfidence_thresholdÚcalibration_datasetÚbootstrap_samplesÚbootstrap_seedÚ	referencec               ó¤
  ‡&— ddl m}
 ddlm} d|cxk    rdk    r	n n|dk     rt	          d¦  «        ‚|                      ¦   «                              |¦  «         |� t          t          | ||||¦  «        ¦  «        }t          | ||||¦  «        }g g g g f\  }}}Š&t          |d¦  «        D �]\  }\  }}}}| 
                    ||j        d         |j        d         |j        dœ¦  «         |j        D ]S}| 
                    t          |¦  «        dz   |d|j        |j        |j        |j        g|j        |j        z  ddœ¦  «         ŒTt'          |d	g ¦  «        D ]T}| 
                    t          |¦  «        dz   |d|j        |j        |j        |j        g|j        |j        z  ddd
œ¦  «         ŒU|D ]0\  }}|\  }}}}| 
                    |d||||z
  ||z
  g|dœ¦  «         Œ1t)          ||j        |t'          |d	d¦  «        ¦  «        \  }}}‰& 
                    |j        |j        |j        d         |j        d         t/          t'          t'          |dd¦  «        dg ¦  «        ¦  «        d„ t'          |d	g ¦  «        D ¦   «         t'          |dt'          |dd¦  «        ¦  «        d„ |j        D ¦   «         d„ |D ¦   «         |||dœ¦  «         �Œt1          ‰&¦  «        } | d         | d         | d         |dk    r| d         nd| d         | d         | d         ||�dnd|	t3          d„ ‰&D ¦   «         ¦  «        ‰&dœ}!ˆ&fd „|!d!         D ¦   «         |!d"<   ˆ&fd#„t3          d$„ ‰&D ¦   «         ¦  «        D ¦   «         |!d%<   d&d'hd(d)hh d*£d+œ}"ˆ&fd,„|"                     ¦   «         D ¦   «         |!d-<   |rLt7          ‰&||¦  «        |!d.<   t          |!d!         ¦  «        |!d/<   t          |!d!         ¦  «        d0k     rd1nd|!d2<   |r|s|!                     dddd3œ¦  «         |!S |s|!                     d4d4d4d3œ¦  «         |!S  |
¦   «         }#||dd5d6œgi g d7œ|#_        t=          t?          j         ¦   «         ¦  «        5  |# !                    ¦   «          |# "                    |¦  «        }$ ||#|$d8¦  «        }%|% #                    ¦   «          |% $                    ¦   «          |% %                    ¦   «          ddd¦  «         n# 1 swxY w Y   |!                     tM          |%j'        d         ¦  «        tM          |%j'        d0         ¦  «        tM          |%j'        d         ¦  «        d3œ¦  «         |!S )9a?  Evaluate AP and rates; calibrate a threshold only on validation data.

    Pass a validation ``calibration_dataset`` to maximize its F1 before scoring this
    dataset. ``bootstrap_samples`` resamples scene groups for operational-rate CIs.
    Undefined precision, recall, F1, or AP values are represented by None.
    r   r¾   rÀ   r   z6invalid confidence threshold or bootstrap sample countNrÃ   rÇ   rh   rÌ   rÏ   rN   ÚsceneÚtagsc                óB   — g | ]}|j         |j        |j        |j        g‘ŒS rN   ©r@   rA   rC   rD   )r   Úrs     r   r   zevaluate.<locals>.<listcomp>Ü   s)   € Ð(nÐ(nÐ(nÈ¨!¬#¨a¬c°!´'¸!¼(Ð)CÐ(nÐ(nÐ(nr   ÚdomainÚunknownc                óB   — g | ]}|j         |j        |j        |j        g‘ŒS rN   r  )r   rE   s     r   r   zevaluate.<locals>.<listcomp>Þ   s?   € ð +@ð +@ð +@Ø/1ð -/¬D°"´$¸¼À"Ä)Ð+Lð +@ð +@ð +@r   c                ó   — g | ]
\  }}||d œ‘ŒS ))rÎ   rk   rN   rŽ   s      r   r   zevaluate.<locals>.<listcomp>à   s:   € ð '@ð '@ð '@Ù+5¨3°ð 14¸eÐ'DÐ'Dð '@ð '@ð '@r   )rÂ   rì   rC   rD   r  rh   r  rÆ   r–   ri   rj   r{   r   r|   r€   r£   r‚   r�   rƒ   Ú
validationÚfixedc                ó   — h | ]
}|d          ’ŒS ©rì   rN   rv   s     r   ú	<setcomp>zevaluate.<locals>.<setcomp>í   s   € Ð"EÐ"EÐ"E°s 3 z¤?Ð"EÐ"EÐ"Er   )r   r|   r€   zrecall_at_0.25_iou_0.5zrecall_at_threshold_iou_0.5r�   rƒ   r  Úthreshold_sourcer  Ú	group_idsÚ	per_framec                óJ   •‡— i | ]Š‰t          ˆfd „‰D ¦   «         ¦  «        “ŒS )c                ó,   •— g | ]}|d          ‰k    ¯|‘ŒS r  rN   ©r   r'   rØ   s     €r   r   z'evaluate.<locals>.<dictcomp>.<listcomp>ï   s(   ø€ Ð(ZÐ(ZÐ(Z°À#ÀjÄ/ÐUYÒBYÐBY¨ÐBYÐBYÐBYr   )r…   ©r   rØ   Úframess    @€r   rî   zevaluate.<locals>.<dictcomp>ï   sI   øø€ ð <ð <ð <Ø#ð  ¥Ð(ZÐ(ZÐ(ZÐ(Z¸Ð(ZÑ(ZÔ(ZÑ![Ô![ð <ð <ð <r   r  Ú	per_scenec                ó„   •‡— i | ];Š‰i t          ˆfd „‰D ¦   «         ¦  «        ¥t          ˆfd„‰D ¦   «         ¦  «        ¥“Œ<S )c                ó,   •— g | ]}|d          ‰k    ¯|‘ŒS ©r  rN   r   s     €r   r   z'evaluate.<locals>.<dictcomp>.<listcomp>ñ   s(   ø€ Ð,\Ð,\Ð,\°SÀcÈ(ÄmÐW[ÒF[ÐF[¨SÐF[ÐF[ÐF[r   c                ó,   •— g | ]}|d          ‰k    ¯|‘ŒS r&  rN   r   s     €r   r   z'evaluate.<locals>.<dictcomp>.<listcomp>ò   s(   ø€ Ð-[Ð-[Ð-[°cÀsÈ8Ä}ÐVZÒGZÐGZ¨cÐGZÐGZÐGZr   ©r…   rê   r!  s    @€r   rî   zevaluate.<locals>.<dictcomp>ñ   s�   øø€ ð Sð Sð Sà $ð !ð #^¥VÐ,\Ð,\Ð,\Ð,\¸FÐ,\Ñ,\Ô,\Ñ%]Ô%]ð #^Ý$,Ð-[Ð-[Ð-[Ð-[¸VÐ-[Ñ-[Ô-[Ñ$\Ô$\ð#^ð Sð Sð Sr   c                ó   — h | ]
}|d          ’ŒS r&  rN   rv   s     r   r  zevaluate.<locals>.<setcomp>ó   s   € Ð/PÐ/PÐ/PÀ#°°H´Ð/PÐ/PÐ/Pr   Ú
per_domainÚstackedÚstacked_palletsÚoccludedÚpartially_occluded>   Ú	truncatedÚedge_truncatedÚtarget_truncatedÚpartially_truncated)r+  r-  r/  c                óŠ   •‡— i | ]>\  }Š|i t          ˆfd „‰D ¦   «         ¦  «        ¥t          ˆfd„‰D ¦   «         ¦  «        ¥“Œ?S )c                óD   •— g | ]}t          |d          ¦  «        ‰z  ¯|‘ŒS ©r  ©rb   ©r   r  r  s     €r   r   z'evaluate.<locals>.<dictcomp>.<listcomp>ö   s.   ø€ Ð,VÐ,VÐ,V°1Å#ÀaÈÄiÁ.Ä.ÐQUÑBUÐ,V¨QÐ,VÐ,VÐ,Vr   c                óD   •— g | ]}t          |d          ¦  «        ‰z  ¯|‘ŒS r5  r6  r7  s     €r   r   z'evaluate.<locals>.<dictcomp>.<listcomp>÷   s.   ø€ Ð-WÐ-WÐ-W°AÅ3ÀqÈÄyÁ>Ä>ÐRVÑCVÐ-W¨aÐ-WÐ-WÐ-Wr   r(  )r   rØ   r  r"  s     @€r   rî   zevaluate.<locals>.<dictcomp>ö   s   øø€ ð ?ð ?ð ?á!*  dð "ð #Z¥VÐ,VÐ,VÐ,VÐ,V¸Ð,VÑ,VÔ,VÑ%WÔ%Wð #ZÝ$,Ð-WÐ-WÐ-WÐ-W¸Ð-WÑ-WÔ-WÑ$XÔ$Xð#Zð ?ð ?ð ?r   Úper_attributeÚbootstrap_ciÚbootstrap_groupsr<   zWFewer than two independent groups; interval does not quantify between-session variationÚbootstrap_warningrÒ   r>   rÖ   r×   )r   r   rÙ   rÚ   ÚlicensesrÊ   )(rÛ   r¿   rÜ   rÁ   r/   Úevalr�   r¼   r™   r-   r3   Úoriginal_sizer   r:   r.   r@   rA   rC   rD   r¬   rn   rÂ   rì   rù   r…   rc   rü   r  Úupdater’   r   rÝ   rÞ   rß   rà   rá   râ   rã   r%   rä   )'r   r’   r‰   r‡   rˆ   r  r  r	  r
  r  r¿   rÁ   r”   r   rf   r–   rÈ   r—   r'   r   r:   rE   rq   r&   rk   rF   rG   rH   rI   ri   rj   r{   r  rè   Úaliasesrç   Ú	pred_cocoré   r"  s'                                         @r   rá   rá   ´   ss  ø€ ð &Ð%Ð%Ð%Ð%Ð%Ø-Ð-Ð-Ð-Ð-Ð-àÐ$Ð)Ð)Ò)Ð)¨Ò)Ð)Ð)Ð)Ð)Ð->ÀÒ-BÐ-BÝÐQÑRÔRÐRØ	‡J‚J�L„L‡O‚O�FÑÔÐØÐ&Ý0Ý�UÐ/°¸ÀWÑMÔMñ Oô  OÐå˜ ¨°¸WÑEÔE€IØ02°B¸¸B°Ñ-€FˆL˜+ vÝ09¸)ÀQÑ0GÔ0Gð 6ñ 6Ñ,ˆÑ,�6˜3  eØ�Š˜X°Ô0BÀ1Ô0EØ!%Ô!3°AÔ!6ÀSÄYðPð Pñ 	Qô 	Qð 	Qà”)ð 	Nð 	NˆBØ×Ò¥s¨<Ñ'8Ô'8¸1Ñ'<È(Ø01¸B¼DÀ"Ä$ÈÌÐRTÔR[Ð;\Ø)+¬°B´IÑ)=È!ð!Mð !Mñ Nô Nð Nð Nõ ˜cÐ"2°2Ñ6Ô6ð 	\ð 	\ˆFØ×Ò¥c¨,Ñ&7Ô&7¸Ñ&9ÀXÐ\]Ø)/¬°&´(¸6¼<ÈÌÐ(VØ(.¬°V´]Ñ(BÈQÐXYð![ð ![ñ \ô \ð \ð \ð  ð 	Uð 	U‰JˆC�Ø ‰NˆB��B˜Ø×Ò¨HÀQØ)+¨R°°b±¸"¸r¹'Ð(BÈUð Tð  Tñ Uô Uð Uð Uå# E¨3¬9Ð6JÍ7ÐSVÐWgÐhjÑKkÔKkÑlÔl‰
ˆˆB�Ø�Š 6¤?ÀÄØ#Ô1°!Ô4¸dÔ>PÐQRÔ>SÝ"¥7­7°3°w¸tÑ+DÔ+DÀVÈBÑ#OÔ#OÑPÔPØ(nÐ(nÍWÐUXÐYiÐjlÑMmÔMmÐ(nÑ(nÔ(nÝ!(¨¨h½ÀÈÐR[Ñ8\Ô8\Ñ!]Ô!]ð+@ð +@Ø58´Yð+@ñ +@ô +@ð'@ð '@Ø9>ð'@ñ '@ô '@à r°ð	5ð 	5ñ 		6ô 		6ð 		6ñ 		6õ �6‰NŒN€EØ˜hœ°E¸)Ô4DØ+0Ð1LÔ+Mà%¨Ò-Ð-ð ).¨h¬¨Ø37Ø-2°8¬_Ø  Ô-°U¸4´[Ø&:Ø2EÐ2Q , ,ÐW^Ø$Ý!Ð"EÐ"E¸fÐ"EÑ"EÔ"EÑFÔFØ!ð
#ð 
#€Fð<ð <ð <ð <Ø'-¨kÔ':ð<ñ <ô <€Fˆ;ÑðSð Sð Sð Så(.Ð/PÐ/PÈÐ/PÑ/PÔ/PÑ(QÔ(QðSñ Sô S€Fˆ<Ñð "Ð"3Ð4ÀÐL`Ð@aØ`Ð`Ð`ðbð b€Gð?ð ?ð ?ð ?à.5¯mªm©o¬oð?ñ ?ô ?€Fˆ?Ñð ð fÝ!+¨FÐ4EÀ~Ñ!VÔ!Vˆˆ~ÑÝ#& v¨kÔ':Ñ#;Ô#;ˆÐ!Ñ"õ BEð  FLð  MXô  FYñ  BZô  BZð  [\ò  B\ð  B\Ð$}Ð$}ð  bfˆÐ"Ñ#Øð ˜ð Ø�Š˜t¨T¸dÐCÐCÑDÔDÐDØˆØð Ø�Š˜s¨C¸CÐ@Ð@ÑAÔAÐAØˆØˆ4‰6Œ6€DØ$°\Ø*+°XÐ#>Ð#>Ð"?ÈÐY[ð]ð ]€D„Lå	�œ™œÑ	'Ô	'ð ð Ø×ÒÑÔÐØ—L’L Ñ-Ô-ˆ	Ø�H˜T 9¨fÑ5Ô5ˆ	Ø×ÒÑÔÐØ×ÒÑÔÐØ×ÒÑÔÐðð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð ð ‡M‚M�5 ¤°Ô!3Ñ4Ô4½eÀIÄOÐTUÔDVÑ>WÔ>WÝ# I¤O°AÔ$6Ñ7Ô7ð9ð 9ñ :ô :ð :à€Ms   Ñ%A3S$Ó$S(Ó+S()rN   r¦   )Ú__doc__Ú
__future__r   Ú
contextlibr   rÝ   r2   r¯   r+   Útorch.utils.datar   r’   r   r   Úloadingr	   r   r   rM   rn   re   r…   r™   r¼   rê   r  rá   rN   r   r   ú<module>rH     s™  ðØ MÐ Mà "Ð "Ð "Ð "Ð "Ð "à &Ð &Ð &Ð &Ð &Ð &Ø 	€	€	€	à Ð Ð Ð Ø €€€Ø 'Ð 'Ð 'Ð 'Ð 'Ð 'à 2Ð 2Ð 2Ð 2Ð 2Ð 2Ð 2Ð 2Ø  Ð  Ð  Ð  Ð  Ð  ð /4¸#ð Nð Nð Nð Nð Nð :?ÀSð ð ð ð ð ð06ð 6ð 6ð*ð *ð *ð *ð"ð ð ð
@ð 
@ð 
@ðð ð ð ðð ð ð6sð sð sð4Dð Dð Dð( (0¸AÀqØ"&¸DØ °ØCð[ð [ð [ð [ð [ð [ð [r   