§
    9²¸j4  ã                   ó˜   — d Z ddlmZ ddlmZ ddlmZ ddlmZ ddl	m
Z
  e
j        e¦  «        Zdd	iZg d
¢Z G d„ de¦  «        ZdgZdS )zTokenization classes for ESMC.é    )Ú	Tokenizer)ÚBPE)ÚTemplateProcessingé   )ÚTokenizersBackend)ÚloggingÚtokenizer_fileztokenizer.json)!ú<cls>ú<pad>ú<eos>ú<unk>ÚLÚAÚGÚVÚSÚEÚRÚTÚIÚDÚPÚKÚQÚNÚFÚYÚMÚHÚWÚCÚXÚBÚUÚZÚOú.ú-ú|ú<mask>c                   óD   ‡ — e Zd ZdZeZddgZeZ	 	 	 	 	 	 	 	 dˆ fd„	Z	ˆ xZ
S )ÚEsmcTokenizera©  
    Construct an ESMC tokenizer.

    This tokenizer is a character-level tokenizer backed by the HuggingFace `tokenizers` library.
    It wraps every sequence with ``<cls>`` and ``<eos>`` tokens and supports a ``|`` chain-break
    token for multi-chain inputs.

    Args:
        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token.
        cls_token (`str`, *optional*, defaults to `"<cls>"`):
            The classification token (prepended to every sequence).
        pad_token (`str`, *optional*, defaults to `"<pad>"`):
            The padding token.
        mask_token (`str`, *optional*, defaults to `"<mask>"`):
            The mask token, used for masked language modelling.
        eos_token (`str`, *optional*, defaults to `"<eos>"`):
            The end-of-sequence token (appended to every sequence).
        bos_token (`str`, *optional*, defaults to `"<cls>"`):
            The beginning-of-sequence token (prepended to every sequence). When unset, uses cls_token.
        chain_break_token (`str`, *optional*, defaults to `"|"`):
            Token inserted between chains in multi-chain protein inputs. Registered as a model-specific
            special token, so it is exposed as `chain_break_token` / `chain_break_token_id`.
        extra_special_tokens (`list[str]`, *optional*):
            Additional special tokens to register with the tokenizer (round-tripped from a saved config).

    Examples:

    ```python
    >>> from transformers import EsmcTokenizer

    >>> tokenizer = EsmcTokenizer()
    >>> tokenizer("ACDEFGHIKLMNPQRSTVWY")["input_ids"]
    [0, 5, 23, 13, 9, 18, 6, 21, 12, 15, 4, 20, 17, 14, 16, 10, 8, 11, 7, 22, 19, 2]
    ```
    Ú	input_idsÚattention_maskr   r
   r   r*   r   Nr)   c	                 ó’  •— d„ t          t          ¦  «        D ¦   «         }
t          t          |
g t	          |¦  «        ¬¦  «        ¦  «        | _         t          ¦   «         j        d||€|n|||||||dœ|	¤Ž t          | j	        › d| j
        › �| j	        |
| j	                 f| j
        |
| j
                 fg¬¦  «        | j        _        d S )Nc                 ó   — i | ]\  }}||“Œ	S © r1   )Ú.0ÚindÚtoks      úz/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/esmc/tokenization_esmc.pyú
<dictcomp>z*EsmcTokenizer.__init__.<locals>.<dictcomp>y   s   € ÐJÐJÐJ¡H C¨�s˜CÐJÐJÐJó    )ÚmergesÚ	unk_token)r9   Ú	bos_tokenÚ	cls_tokenÚ	pad_tokenÚ
mask_tokenÚ	eos_tokenÚchain_break_tokenÚextra_special_tokensz $A )ÚsingleÚspecial_tokensr1   )Ú	enumerateÚSEQUENCE_VOCABr   r   ÚstrÚ
_tokenizerÚsuperÚ__init__r   r;   r>   Úpost_processor)Úselfr9   r;   r<   r=   r>   r:   r?   r@   ÚkwargsÚtoken_to_idÚ	__class__s              €r5   rH   zEsmcTokenizer.__init__m   sô   ø€ ð KÐJµ	½.Ñ0IÔ0IÐJÑJÔJˆõ $¥C¨¸BÍ#ÈiÉ.Ì.Ð$YÑ$YÔ$YÑZÔZˆŒà�‰ŒÔð 	
Øà#,Ð#4�i�i¸)ØØØ!ØØ/Ø!5ð	
ð 	
ð ð	
ð 	
ð 	
õ *<Ø”nÐ:Ð:¨$¬.Ð:Ð:à” ¨T¬^Ô!<Ð=Ø” ¨T¬^Ô!<Ð=ðð*
ñ *
ô *
ˆŒÔ&Ð&Ð&r7   )r   r
   r   r*   r   Nr)   N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚmodel_input_namesr   ÚmodelrH   Ú__classcell__)rM   s   @r5   r,   r,   C   s{   ø€ € € € € ð#ð #ðJ *ÐØ$Ð&6Ð7ÐØ€Eð ØØØØØØØ!ð%
ð %
ð %
ð %
ð %
ð %
ð %
ð %
ð %
ð %
r7   r,   N)rQ   Ú
tokenizersr   Útokenizers.modelsr   Útokenizers.processorsr   Útokenization_utils_tokenizersr   Úutilsr   Ú
get_loggerrN   ÚloggerrR   rD   r,   Ú__all__r1   r7   r5   ú<module>r_      sÛ   ðð %Ð $à  Ð  Ð  Ð  Ð  Ð  Ø !Ð !Ð !Ð !Ð !Ð !Ø 4Ð 4Ð 4Ð 4Ð 4Ð 4à >Ð >Ð >Ð >Ð >Ð >Ø Ð Ð Ð Ð Ð ð 
ˆÔ	˜HÑ	%Ô	%€à%Ð'7Ð8Ð ð"ð "ð "€ðJO
ð O
ð O
ð O
ð O
Ð%ñ O
ô O
ð O
ðd Ð
€€€r7   