§
    Ú³¸j–<  ã                   ó  — d dl Z d dlZd dlZd dlZd dlmZ d dlmZmZm	Z	m
Z
mZmZmZ d dlmZmZ ej                             e¦  «        Zej                             ed¦  «        Z ee¦  «        5 Ze                     ¦   «         Zddd¦  «         n# 1 swxY w Y    G d„ d¦  «        Ze
e         Ze
ee                  Z G d„ d	e¦  «        Z G d
„ d¦  «        Z G d„ d¦  «        Z  G d„ d¦  «        Z!efde
e"         de"fd„Z#dS )é    N)ÚTemplate)ÚAnyÚCallableÚDictÚListÚ
NamedTupleÚOptionalÚTuple)ÚEncodingÚ	Tokenizerzvisualizer-styles.cssc                   óB   — e Zd ZU eed<   eed<   eed<   dededefd„ZdS )Ú
AnnotationÚstartÚendÚlabelc                 ó0   — || _         || _        || _        d S ©N)r   r   r   )Úselfr   r   r   s       úk/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/tokenizers/tools/visualizer.pyÚ__init__zAnnotation.__init__   s   € ØˆŒ
ØˆŒØˆŒ
ˆ
ˆ
ó    N)Ú__name__Ú
__module__Ú__qualname__ÚintÚ__annotations__Ústrr   © r   r   r   r      sZ   € € € € € € Ø€J€J�JØ	€H€H�HØ€J€J�Jð˜cð ¨ð °Cð ð ð ð ð ð r   r   c                   ó<   — e Zd ZU ee         ed<   ee         ed<   dS )ÚCharStateKeyÚtoken_ixÚanno_ixN)r   r   r   r	   r   r   r   r   r   r    r       s2   € € € € € € Ø�sŒmÐÐÑØ�cŒ]ÐÐÑÐÐr   r    c                   ód   — e Zd ZU ee         ed<   d„ Zed„ ¦   «         Zed„ ¦   «         Z	de
fd„ZdS )Ú	CharStateÚchar_ixc                 ó0   — || _         d | _        g | _        d S r   )r%   r"   Útokens)r   r%   s     r   r   zCharState.__init__'   s   € ØˆŒà&*ˆŒØ!#ˆŒˆˆr   c                 óP   — t          | j        ¦  «        dk    r| j        d         nd S )Nr   ©Úlenr'   ©r   s    r   r!   zCharState.token_ix-   s&   € å!$ T¤[Ñ!1Ô!1°AÒ!5Ð!5ˆtŒ{˜1Œ~ˆ~¸4Ð?r   c                 ó2   — t          | j        ¦  «        dk    S )zJ
        BPE tokenizers can output more than one token for a char
        é   r)   r+   s    r   Úis_multitokenzCharState.is_multitoken1   s   € õ
 �4”;ÑÔ !Ò#Ð#r   Úreturnc                 ó8   — t          | j        | j        ¬¦  «        S )N)r!   r"   )r    r!   r"   r+   s    r   Úpartition_keyzCharState.partition_key8   s#   € ÝØ”]Ø”Lð
ñ 
ô 
ð 	
r   N)r   r   r   r	   r   r   r   Úpropertyr!   r.   r    r1   r   r   r   r$   r$   $   s‰   € € € € € € Ø�cŒ]ÐÐÑð$ð $ð $ð ð@ð @ñ „Xð@ð ð$ð $ñ „Xð$ð
˜|ð 
ð 
ð 
ð 
ð 
ð 
r   r$   c                   ó   — e Zd ZdS )ÚAlignedN)r   r   r   r   r   r   r4   r4   ?   s   € € € € € Ø€Dr   r4   c            
       ó¦  — e Zd ZdZ ej        dej        ¬¦  «        Z	 	 ddede	de
eegef                  fd	„Z	 	 dd
ede
ee                  de
e	         de
e         fd„Zededeeef         fd„¦   «         Zedee         d
edefd„¦   «         Zed
edededefd„¦   «         Zed
ededefd„¦   «         Zed
edededee         fd„¦   «         ZdS )ÚEncodingVisualizera³  
    Build an EncodingVisualizer

    Args:

         tokenizer (:class:`~tokenizers.Tokenizer`):
            A tokenizer instance

         default_to_notebook (:obj:`bool`):
            Whether to render html output in a notebook by default

         annotation_converter (:obj:`Callable`, `optional`):
            An optional (lambda) function that takes an annotation in any format and returns
            an Annotation object
    z(.{1})?(unk|oov)(.{1})?)ÚflagsTNÚ	tokenizerÚdefault_to_notebookÚannotation_converterc                 ó¼   — |rD	 ddl m}m} n:# t          $ r- 	 ddlm}m} n # t          $ r d}t          |¦  «        d ‚w xY wY nw xY w|| _        || _        || _        d S )Nr   ©ÚHTMLÚdisplayú–We couldn't import IPython utils for html display.
Are you running in a notebook?
You can also pass `default_to_notebook=False` to get back raw HTML.
)ÚIPython.displayr=   r>   ÚImportErrorÚIPython.core.displayr8   r9   Úannotation_coverter)r   r8   r9   r:   r=   r>   Úmsgs          r   r   zEncodingVisualizer.__init__V   sÆ   € ð ð 	5ð5Ø9Ð9Ð9Ð9Ð9Ð9Ð9Ð9Ð9øÝð 	5ð 	5ð 	5ð5ØBÐBÐBÐBÐBÐBÐBÐBÐBøÝ"ð 5ð 5ð 5ð`ð õ
 & cÑ*Ô*°Ð4ð5øøøð CÐBð	5øøøð #ˆŒØ#6ˆÔ Ø#7ˆÔ Øˆs!   „ �
A˜! A¡>¾AÁAÚtextÚannotationsr/   c                 óž  — | j         }|�|}|rD	 ddlm}m} n:# t          $ r- 	 ddlm}m} n # t          $ r d}t	          |¦  «        d‚w xY wY nw xY w|€g }| j        �"t          t          | j        |¦  «        ¦  «        }| j	         
                    |¦  «        }t                               |||¦  «        }	|r | ||	¦  «        ¦  «         dS |	S )aß  
        Build a visualization of the given text

        Args:
            text (:obj:`str`):
                The text to tokenize

            annotations (:obj:`List[Annotation]`, `optional`):
                An optional list of annotations of the text. The can either be an annotation class
                or anything else if you instantiated the visualizer with a converter function

            default_to_notebook (:obj:`bool`, `optional`, defaults to `False`):
                If True, will render the html in a notebook. Otherwise returns an html string.

        Returns:
            The HTML string if default_to_notebook is False, otherwise (default) returns None and
            renders the HTML in the notebook

        Nr   r<   r?   )r9   r@   r=   r>   rA   rB   rC   ÚlistÚmapr8   Úencoder6   Ú_EncodingVisualizer__make_html)
r   rE   rF   r9   Úfinal_default_to_notebookr=   r>   rD   ÚencodingÚhtmls
             r   Ú__call__zEncodingVisualizer.__call__n   sD  € ð2 %)Ô$<Ð!ØÐ*Ø(;Ð%Ø$ð 	5ð5Ø9Ð9Ð9Ð9Ð9Ð9Ð9Ð9Ð9øÝð 	5ð 	5ð 	5ð5ØBÐBÐBÐBÐBÐBÐBÐBÐBøÝ"ð 5ð 5ð 5ð`ð õ
 & cÑ*Ô*°Ð4ð5øøøð CÐBð	5øøøð ÐØˆKØÔ#Ð/Ý�s 4Ô#;¸[ÑIÔIÑJÔJˆKØ”>×(Ò(¨Ñ.Ô.ˆÝ!×-Ò-¨d°H¸kÑJÔJˆØ$ð 	ØˆG�D�D˜‘J”JÑÔÐÐÐàˆKs#   � ˜
A£,«A¬A	Á	AÁAc                 ó  — t          | ¦  «        dk    ri S t          t          d„ | ¦  «        ¦  «        }t          |¦  «        }t          d|z  ¦  «        }|dk     rd}d}d}d}i }t	          |¦  «        D ]}d|› d	|› d
|› d�||<   ||z  }Œ|S )a  
        Generates a color palette for all the labels in a given set of annotations

        Args:
          annotations (:obj:`Annotation`):
            A list of annotations

        Returns:
            :obj:`dict`: A dictionary mapping labels to colors in HSL format
        r   c                 ó   — | j         S r   )r   )Úxs    r   ú<lambda>z;EncodingVisualizer.calculate_label_colors.<locals>.<lambda>°   s   €  1¤7€ r   éÿ   é   é    é@   é
   zhsl(ú,z%,z%))r*   ÚsetrI   r   Úsorted)	rF   ÚlabelsÚ
num_labelsÚh_stepÚsÚlÚhÚcolorsr   s	            r   Úcalculate_label_colorsz)EncodingVisualizer.calculate_label_colors¢   s¿   € õ ˆ{ÑÔ˜qÒ Ð ØˆIÝ•SÐ*Ð*¨KÑ8Ô8Ñ9Ô9ˆÝ˜‘[”[ˆ
Ý�S˜:Ñ%Ñ&Ô&ˆØ�BŠ;ˆ;ØˆFØˆØˆØˆØˆå˜F‘^”^ð 	ð 	ˆEØ1 1Ð1Ð1 qÐ1Ð1¨AÐ1Ð1Ð1ˆF�5‰MØ�‰KˆAˆAØˆr   Úconsecutive_chars_listrM   c                 ó<  — | d         }|j         €|j        |j                 }d|› d�S | d         }|j         €J ‚|j         €J ‚|j         }|j         dz   }|||…         }g }	i }
|j        �À|	                     d¦  «         |j        r|	                     d¦  «         |j        d	z  r|	                     d
¦  «         n|	                     d¦  «         t
          j                             |j        |j                 ¦  «        �*|	                     d¦  «         |j        |j                 |
d<   n|	                     d¦  «         dd                     |	¦  «        › d�}d}|
 	                    ¦   «         D ]\  }}|d|› d|› d�z  }Œt          j        |¦  «        }d|› d|› d|› d�S )aã  
        Converts a list of "consecutive chars" into a single HTML element.
        Chars are consecutive if they fall under the same word, token and annotation.
        The CharState class is a named tuple with a "partition_key" method that makes it easy to
        compare if two chars are consecutive.

        Args:
            consecutive_chars_list (:obj:`List[CharState]`):
                A list of CharStates that have been grouped together

            text (:obj:`str`):
                The original text being processed

            encoding (:class:`~tokenizers.Encoding`):
                The encoding returned from the tokenizer

        Returns:
            :obj:`str`: The HTML span for a set of consecutive chars
        r   Nz(<span class="special-token" data-stoken=z></span>éÿÿÿÿr-   Útokenzmulti-tokené   z	odd-tokenz
even-tokenzspecial-tokenÚstokz	non-tokenzclass="ú ú"Ú z data-z="z<span z >ú</span>)r%   r'   r!   Úappendr.   r6   Úunk_token_regexÚsearchÚjoinÚitemsrN   Úescape)rd   rE   rM   ÚfirstÚstokenÚlastr   r   Ú	span_textÚcss_classesÚ
data_itemsÚcssÚdataÚkeyÚvals                  r   Úconsecutive_chars_to_htmlz,EncodingVisualizer.consecutive_chars_to_html¿   sï  € ð2 ' qÔ)ˆØŒ=Ð à”_ U¤^Ô4ˆFð O¸fÐNÐNÐNÐNà% bÔ)ˆØŒ}Ð(Ð(Ð(ØŒ|Ð'Ð'Ð'Ø”ˆØŒl˜QÑˆØ˜˜s˜”Oˆ	ØˆØˆ
ØŒ>Ð%à×Ò˜wÑ'Ô'Ð'ØÔ"ð 2Ø×"Ò" =Ñ1Ô1Ð1ØŒ~ Ñ!ð 1ð
 ×"Ò" ;Ñ/Ô/Ð/Ð/ð ×"Ò" <Ñ0Ô0Ð0Ý!Ô1×8Ò8¸¼ÈÌÔ9XÑYÔYÐeà×"Ò" ?Ñ3Ô3Ð3à%-¤_°U´^Ô%D�
˜6Ñ"øð ×Ò˜{Ñ+Ô+Ð+Ø4˜#Ÿ(š( ;Ñ/Ô/Ð4Ð4Ð4ˆØˆØ"×(Ò(Ñ*Ô*ð 	+ð 	+‰HˆC�ØÐ*˜SÐ*Ð* CÐ*Ð*Ð*Ñ*ˆDˆDÝ”K 	Ñ*Ô*ˆ	Ø8˜Ð8Ð8˜dÐ8Ð8 iÐ8Ð8Ð8Ð8r   c                 óþ  — t                                | ||¦  «        }|d         g}|d         j        }g }t                                |¦  «        }|d         j        }|�3||         }	|	j        }
||
         }|                     d|› d|
› d�¦  «         |dd …         D �]
}|j        }||k    r|                     t                                || |¬¦  «        ¦  «         |g}|�|                     d¦  «         |�3||         }	|	j        }
||
         }|                     d|› d|
› d�¦  «         |}|                     ¦   «         |d                              ¦   «         k    r|                     |¦  «         Œ×|                     t                                || |¬¦  «        ¦  «         |g}�Œ|                     t                                || |¬¦  «        ¦  «         |�|                     d¦  «         t          |¦  «        }|S )Nr   z&<span class="annotation" style="color:z" data-label="z">r-   )rE   rM   rm   )	r6   Ú%_EncodingVisualizer__make_char_statesr"   rc   r   rn   r~   r1   ÚHTMLBody)rE   rM   rF   Úchar_statesÚcurrent_consecutive_charsÚprev_anno_ixÚspansÚlabel_colors_dictÚcur_anno_ixÚannor   ÚcolorÚcsÚress                 r   Ú__make_htmlzEncodingVisualizer.__make_html  sY  € å(×;Ò;¸DÀ(ÈKÑXÔXˆØ%0°¤^Ð$4Ð!Ø" 1”~Ô-ˆØˆÝ.×EÒEÀkÑRÔRÐØ! !”nÔ,ˆØÐ"à˜{Ô+ˆDØ”JˆEØ% eÔ,ˆEØ�LŠLÐ`À%Ð`Ð`ÐW\Ð`Ð`Ð`ÑaÔaÐaà˜a˜b˜b”/ð &	1ñ &	1ˆBØœ*ˆKØ˜lÒ*Ð*à—’å&×@Ò@Ø1Ø!Ø!)ð Añ ô ñô ð ð .0¨DÐ)àÐ+à—L’L Ñ+Ô+Ð+ØÐ*à& {Ô3�DØ œJ�EØ-¨eÔ4�EØ—L’LÐ!hÈ%Ð!hÐ!hÐ_dÐ!hÐ!hÐ!hÑiÔiÐiØ&ˆLà×ÒÑ!Ô!Ð%>¸qÔ%A×%OÒ%OÑ%QÔ%QÒQÐQà)×0Ò0°Ñ4Ô4Ð4Ð4ð —’Ý&×@Ò@Ø1Ø!Ø!)ð Añ ô ñô ð ð .0¨DÐ)Ñ)ð 	�ŠÝ×8Ò8Ø)ØØ!ð 9ñ ô ñ	
ô 	
ð 	
ð Ð"Ø�LŠL˜Ñ#Ô#Ð#å�u‰oŒoˆØˆ
r   c                 óš   — dgt          | ¦  «        z  }t          |¦  «        D ]'\  }}t          |j        |j        ¦  «        D ]}|||<   ŒŒ(|S )aÓ  
        Args:
            text (:obj:`str`):
                The raw text we want to align to

            annotations (:obj:`AnnotationList`):
                A (possibly empty) list of annotations

        Returns:
            A list of  length len(text) whose entry at index i is None if there is no annotation on
            character i or k, the index of the annotation that covers index i where k is with
            respect to the list of annotations
        N)r*   Ú	enumerateÚranger   r   )rE   rF   Úannotation_mapr"   ÚaÚis         r   Ú__make_anno_mapz"EncodingVisualizer.__make_anno_mapM  sf   € ð ˜¥# d¡)¤)Ñ+ˆÝ# KÑ0Ô0ð 	,ð 	,‰JˆG�QÝ˜1œ7 A¤EÑ*Ô*ð ,ð ,�Ø$+�˜qÑ!Ð!ð,àÐr   c                 óž  — t                                | |¦  «        }d„ t          t          | ¦  «        ¦  «        D ¦   «         }t	          |j        ¦  «        D ]T\  }}|                     |¦  «        }|�8|\  }}	t          ||	¦  «        D ]"}
||
         j                             |¦  «         Œ#ŒUt	          |¦  «        D ]\  }}|||         _        Œ|S )aÂ  
        For each character in the original text, we emit a tuple representing it's "state":

            * which token_ix it corresponds to
            * which word_ix it corresponds to
            * which annotation_ix it corresponds to

        Args:
            text (:obj:`str`):
                The raw text we want to align to

            annotations (:obj:`List[Annotation]`):
                A (possibly empty) list of annotations

            encoding: (:class:`~tokenizers.Encoding`):
                The encoding returned from the tokenizer

        Returns:
            :obj:`List[CharState]`: A list of CharStates, indicating for each char in the text what
            it's state is
        c                 ó,   — g | ]}t          |¦  «        ‘ŒS r   )r$   )Ú.0r%   s     r   ú
<listcomp>z9EncodingVisualizer.__make_char_states.<locals>.<listcomp>{  s    € Ð'[Ð'[Ð'[¸w­	°'Ñ(:Ô(:Ð'[Ð'[Ð'[r   )	r6   Ú"_EncodingVisualizer__make_anno_mapr�   r*   rŽ   r'   Útoken_to_charsrn   r"   )rE   rM   rF   r�   r‚   r!   rg   Úoffsetsr   r   r’   r%   r"   s                r   Ú__make_char_statesz%EncodingVisualizer.__make_char_statesb  sç   € õ. ,×;Ò;¸DÀ+ÑNÔNˆà'[Ð'[Í%ÕPSÐTXÑPYÔPYÑJZÔJZÐ'[Ñ'[Ô'[ˆÝ(¨¬Ñ9Ô9ð 	;ð 	;‰OˆH�eØ×-Ò-¨hÑ7Ô7ˆGØÐ"Ø$‘
��sÝ˜u cÑ*Ô*ð ;ð ;�AØ ”NÔ)×0Ò0°Ñ:Ô:Ð:Ð:øÝ )¨.Ñ 9Ô 9ð 	3ð 	3ÑˆG�WØ+2ˆK˜Ô Ô(Ð(àÐr   )TN)NN)r   r   r   Ú__doc__ÚreÚcompileÚ
IGNORECASEro   r   Úboolr	   r   r   r   r   r   r   rO   ÚstaticmethodÚAnnotationListr   rc   r$   r   r~   rK   ÚPartialIntListr˜   r€   r   r   r   r6   r6   C   s  € € € € € ðð ð  !�b”jÐ!>ÀbÄmÐTÑTÔT€Oð
 %)ØFJð	ð àðð "ðð ' x°°°zÐ0AÔ'BÔCð	ð ð ð ð6 ,0Ø.2ð	2ð 2àð2ð ˜d 3œiÔ(ð2ð & dœ^ð	2ð
 
�#Œð2ð 2ð 2ð 2ðh ð¨Nð ¸tÀCÈÀH¼~ð ð ð ñ „\ðð8 ðD9Ø $ Y¤ðD9àðD9ð ðD9ð D9ð D9ñ „\ðD9ðL ðD˜#ð D¨ð DÀð DÐSVð Dð Dð Dñ „\ðDðL ð˜cð °ð À>ð ð ð ñ „\ðð( ð" ð "°ð "À~ð "ÐZ^Ð_hÔZið "ð "ð "ñ „\ð"ð "ð "r   r6   Úchildrenr/   c                 ó>   — d                      | ¦  «        }d|› d|› d�S )a[  
    Generates the full html with css from a list of html spans

    Args:
        children (:obj:`List[str]`):
            A list of strings, assumed to be html elements

        css_styles (:obj:`str`, `optional`):
            Optional alternative implementation of the css

    Returns:
        :obj:`str`: An HTML string with style markup
    rl   z?
    <html>
        <head>
            <style>
                zs
            </style>
        </head>
        <body>
            <div class="tokenized-text" dir=auto>
            z4
            </div>
        </body>
    </html>
    )rq   )r¤   Ú
css_stylesÚchildren_texts      r   r�   r�   ˆ  sB   € ð —G’G˜HÑ%Ô%€Mðð ð	ð ð ðð ð ð r   )$rN   Ú	itertoolsÚosr�   Ústringr   Útypingr   r   r   r   r   r	   r
   Ú
tokenizersr   r   ÚpathÚdirnameÚ__file__rq   Úcss_filenameÚopenÚfÚreadrz   r   r¢   r   r£   r    r$   r4   r6   r   r�   r   r   r   ú<module>r´      sK  ðØ €€€Ø Ð Ð Ð Ø 	€	€	€	Ø 	€	€	€	Ø Ð Ð Ð Ð Ð Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ Ià *Ð *Ð *Ð *Ð *Ð *Ð *Ð *à
Œ'�/Š/˜(Ñ
#Ô
#€ØŒw�|Š|˜GÐ%<Ñ=Ô=€Ø	€Tˆ,ÑÔð ˜1Ø
�&Š&‰(Œ(€Cðð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð ðð ð ð ð ñ ô ð ð �jÔ!€Ø�h˜s”mÔ$€ðð ð ð ð �:ñ ô ð ð

ð 
ð 
ð 
ð 
ñ 
ô 
ð 
ð6	ð 	ð 	ð 	ð 	ñ 	ô 	ð 	ðBð Bð Bð Bð Bñ Bô Bð BðJ
 .1ð ð �t˜C”yð °Sð ð ð ð ð ð s   Á1BÂBÂB