§
    9²¸j” ã                   óÚ  — d Z ddlZddlZddlZddlmZ ddlmZ ddlm	Z	 ddl
mZ ddlmZ ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZ ddlmZm Z m!Z!m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z( ddl)m*Z* ddl+m,Z, ddl-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4 ddl5m6Z6m7Z7m8Z8m9Z9  e9j:        e;¦  «        Z<dZ=dZ>dZ?dZ@dZAdZBe.dz  Z.ee e!e"dœZCe=e@dœZD e7e.¦  «         G d„ d e1¦  «        ¦   «         ZEeEZFdS )!z‘
Tokenization classes for fast tokenizers (provided by HuggingFace's tokenizers library). For slow (python) tokenizers
see tokenization_utils.py
é    N)Údefaultdict)ÚIterable)Úcopyfile)ÚAny)Úis_offline_mode)Ú
AddedTokenÚ
processors)ÚEncoding)Ú	Tokenizer)ÚDecoder)ÚBPEÚUnigram)Ú
BpeTrainerÚUnigramTrainerÚWordLevelTrainerÚWordPieceTrainer)Úcached_fileé   )ÚSpmConverter)Úconvert_gguf_tokenizer)Úis_tekken_vocab_filename)Úload_gguf_checkpoint)ÚINIT_TOKENIZER_DOCSTRINGÚBatchEncodingÚPreTokenizedInputÚPreTrainedTokenizerBaseÚ	TextInputÚTruncationStrategyÚgenerate_merges)ÚPaddingStrategyÚadd_end_docstringsÚhf_apiÚloggingztokenizer.jsonzspecial_tokens_map.jsonztokenizer_config.jsonztokenizer.modelzadded_tokens.jsonztiktoken.modelu¡  
        tokenizer_object ([`tokenizers.Tokenizer`]):
            A [`tokenizers.Tokenizer`] object from ðŸ¤— tokenizers to instantiate from. See [Using tokenizers from ðŸ¤—
            tokenizers](../fast_tokenizers) for more information.
        tokenizer_file ([`str`]):
            A path to a local JSON file representing a previously serialized [`tokenizers.Tokenizer`] object from ðŸ¤—
            tokenizers.
)r   r   Ú	WordLevelÚ	WordPiece)Útokenizer_fileÚ
vocab_filec            )       ó’  ‡ — e Zd ZdZeZdZdZedSd„¦   «         Z	ede
dee
ef         dee
ef         fd„¦   «         Zede
dee
ef         dee
ef         fd	„¦   «         Zˆ fd
„Zedefd„¦   «         Zedefd„¦   «         ZdTde
de
dz  dee
         fd„Z	 	 	 	 dUde
ej        z  dedz  de
dz  dede
dz  dee
df         fˆ fd„Zd„ Zed„ ¦   «         Zed„ ¦   «         Zej        d„ ¦   «         Zej        d„ ¦   «         Zd„ Zedefd„¦   «         Zdee
ef         fd„Z edee
ef         fd„¦   «         Z!edee
ef         fd„¦   «         Z"edeee#f         fd„¦   «         Z$e"Z%e$Z&dee
ef         fd „Z'defd!„Z(defd"„Z)ede*fd#„¦   «         Z+ede,fd$„¦   «         Z-	 	 	 	 	 	 	 dVd&e.d'edz  d(edz  d)ed*ed+ed,ed-edeee
ef         e/e.         f         fd.„Z0d/e
defd0„Z1d1ede
dz  fd2„Z2dSd3e/e
e#z           defd4„Z3dSd5edefd6„Z4dSd7ee/e         z  d8ede
e/e
         z  fd9„Z5dWd:e
d5e
dz  d;ede/e
         fd<„Z6d=e7d>e8d?ed@edAedz  dBe
dz  fdC„Z9dd%e7j:        e8j;        ddDddddddddddd%dfd:e<e=z  e/e<         z  e/e=         z  dEe<e=z  e/e<         z  e/e=         z  dz  d;ed=e7d>e8d?edz  d@edFedAedz  dBe
dz  dGedz  d'edz  d(edz  d)ed*ed+ed,ed-edHedz  de>f(dI„Z?dJe/e
         de
fdK„Z@	 	 dXdLee/e         z  d8edMedz  de
fdN„ZA	 	 dYde
ej        z  dOee
df         dedz  de
dz  dee
df         f
dP„ZB	 	 	 dZdQ„ZCe	 	 	 	 	 	 	 d[dR„¦   «         ZDˆ xZES )\ÚTokenizersBackendaQ  
    Base class for all fast tokenizers (wrapping HuggingFace tokenizers library).

    Inherits from [`~tokenization_utils_base.PreTrainedTokenizerBase`].

    Handles all the shared methods for tokenization and special tokens, as well as methods for
    downloading/caching/loading pretrained tokenizers, as well as adding tokens to the vocabulary.

    This class also contains the added tokens in a unified way on top of all tokenizers so we don't have to handle the
    specific vocabulary augmentation methods of the various underlying dictionary structures (BPE, sentencepiece...).
    NFc                 ón  ‡— t          |¦  «        }|                     dd¦  «        }|�Lt          j                             |¦  «        r-| t
          u sd| j        vs|rt          j        |¦  «        |d<   |S |��nt          j                             |¦  «        �rNt          |d¬¦  «        5 }t          j        |¦  «        }ddd¦  «         n# 1 swxY w Y   |                     di ¦  «                             d¦  «        }|d	vret          |¦  «        }t          |d         ¦  «        }	i |	d
<   |dk    rg |	d<   |	|d<   g |d<   t          j        t          j        |¦  «        ¦  «        }
nt          j        |¦  «        }
|
j        |d<   |
j        |d<   |
j        |d<   |
j        �
|
j        |d<   |
j        �
|
j        |d<   |                     d¦  «        }|r�|                     dd¦  «        dk    r	|d         }nt%          |t&          ¦  «        s|g}|D ]C}|                     d¦  «        dk    r(d|v r$ddl}|                     |d         ¦  «        |d<    nŒD|                     di ¦  «                             d
d¦  «        }| j        €8t%          |t&          ¦  «        r"t'          t/          t0          |¦  «        ¦  «        }nÎ| j        j        dk    rFt%          |t&          ¦  «        r0|r.t%          |d         t&          t0          f¦  «        rd„ |D ¦   «         }nx| j        j        dk    rd„ t5          |¦  «        D ¦   «         }nN| j        j        dk    s| j        j        dk    r.t%          |t&          ¦  «        rd„ t5          |¦  «        D ¦   «         }||d
<   t7          | dd¦  «        }d|                     di ¦  «        v r,|r*|j        dk    r|d         d         }d „ |D ¦   «         }||d<   |S |                     d!¦  «        }|                     d"¦  «        }|                     d
¦  «        }|                     d¦  «        }t%          |t8          ¦  «        rXt          j                             |¦  «        r9t;          |¦  «        r*d#d$lm}  ||¦  «        }|                      ¦   «         |d<   |S t%          |t8          ¦  «        rÅt          j                             |¦  «        r¦| !                    d%¦  «        r‘t          j         "                    |¦  «        tF          k    rQ	 |  $                    ||¦  «        }|S # tJ          $ r+}tL           '                    d&|› d'|› d(�¦  «         Y d}~nd}~ww xY w|  (                    ||¦  «        }|S |€At%          |t8          ¦  «        r,t          j                             |¦  «        r||d
<   |d
         }|€At%          |t8          ¦  «        r,t          j                             |¦  «        r||d<   |d         }|€¬| j        �¥| j        j        dk    r•t%          |t           ¦  «        r€d)tR          tT                   d*t&          t8                   fˆfd+„Šg d,¢}tW          ¦   «         }|D ]+}||v r%| ,                     ‰||         g¦  «        ¦  «         Œ,t[          ||¬-¦  «        }||d<   |S ).zª
        Build a `tokenizers.Tokenizer` backend from the available serialization files (tokenizer.json, sentencepiece
        models, tekken.json, vocab/merges).
        r&   NÚ__init__Útokenizer_objectúutf-8©ÚencodingÚmodelÚtype)Nr   Úvocabr   ÚmergesÚadded_tokensÚpost_processorÚtokenizer_paddingÚtokenizer_truncationÚ_json_truncationÚ_json_paddingÚ
normalizerÚSequenceÚnormalizersÚPrecompiledÚprecompiled_charsmapr   Ú_spm_precompiled_charsmapr   c                 ó,   — g | ]}t          |¦  «        ‘ŒS © )Útuple)Ú.0Úitems     úz/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/tokenization_utils_tokenizers.pyú
<listcomp>z>TokenizersBackend.convert_to_native_format.<locals>.<listcomp>¹   s   € Ð;Ð;Ð;¨T�U 4™[œ[Ð;Ð;Ð;ó    r$   c                 ó   — i | ]\  }}||“Œ	S rA   rA   ©rC   ÚiÚtokens      rE   ú
<dictcomp>z>TokenizersBackend.convert_to_native_format.<locals>.<dictcomp>»   s   € ÐCÐCÐC¡h a¨˜ ÐCÐCÐCrG   r%   c                 óT   — i | ]%\  }}t          |t          ¦  «        r|d          n||“Œ&S ©r   )Ú
isinstanceÚlistrI   s      rE   rL   z>TokenizersBackend.convert_to_native_format.<locals>.<dictcomp>¾   s8   € ÐpÐpÐpÑS[ÐSTÐV[­°E½4Ñ)@Ô)@ÐK˜U 1œX˜XÀeÈQÐpÐpÐprG   c                 óš   — g | ]H}t          |t          ¦  «        r"t          |                     d ¦  «        ¦  «        nt          |¦  «        ‘ŒIS )ú )rO   ÚstrrB   Úsplit)rC   Úmerges     rE   rF   z>TokenizersBackend.convert_to_native_format.<locals>.<listcomp>Ä   sK   € ÐrÐrÐrÐbgµZÀÅsÑ5KÔ5KÐ]�% §¢¨CÑ 0Ô 0Ñ1Ô1Ð1ÕQVÐW\ÑQ]ÔQ]ÐrÐrÐrrG   r'   Úmerges_filer   )ÚMistralConverterú.modelz+Could not extract SentencePiece model from z$ using sentencepiece library due to z%. Falling back to TikToken extractor.ÚvaluesÚreturnc                 óÖ   •— g }| D ]b}|€Œt          |t          t          f¦  «        r|                      ‰|¦  «        ¦  «         Œ@|                     t          |¦  «        ¦  «         Œc|S ©N)rO   rP   rB   ÚextendÚappendrS   )rY   Ú	collectedÚvalÚ_iter_special_tokenss      €rE   ra   zHTokenizersBackend.convert_to_native_format.<locals>._iter_special_tokensñ   s{   ø€ Ø')�	Ø!ð 3ð 3�CØ�{Ø Ý! #­­e }Ñ5Ô5ð 3Ø!×(Ò(Ð)=Ð)=¸cÑ)BÔ)BÑCÔCÐCÐCà!×(Ò(­¨S©¬Ñ2Ô2Ð2Ð2Ø Ð rG   )	Ú	pad_tokenÚ	unk_tokenÚ	bos_tokenÚ	eos_tokenÚ	sep_tokenÚ	cls_tokenÚ
mask_tokenÚadditional_special_tokensÚextra_special_tokens)Úskip_tokens).ÚdictÚpopÚosÚpathÚisfiler)   Ú__dict__ÚTokenizerFastÚ	from_fileÚopenÚjsonÚloadÚgetÚfrom_strÚdumpsr5   ÚpaddingÚ
truncationrO   rP   Úbase64Ú	b64decoder0   ÚmaprB   Ú__name__Ú	enumerateÚgetattrrS   r   Úintegrations.mistral.tokenizerrW   Ú	convertedÚendswithÚbasenameÚTIKTOKEN_LEGACY_NAMEÚ_convert_from_sentencepieceÚ	ExceptionÚloggerÚwarningÚ_convert_from_tiktokenr   r   ÚsetÚupdater   )ÚclsÚtrust_remote_codeÚkwargsÚlocal_kwargsÚfast_tokenizer_fileÚtokenizer_handleÚtokenizer_jsonÚ
model_typeÚminimal_tokenizer_jsonÚminimal_modelÚtok_from_fileÚnormalizer_configr:   r|   r2   r3   r'   rV   rW   Ú	converterÚeÚspecial_tokens_keysrk   Úkeyra   s                           @rE   Úconvert_to_native_formatz*TokenizersBackend.convert_to_native_formati   sÂ  ø€ õ ˜F‘|”|ˆØ*×.Ò.Ð/?ÀÑFÔFÐð  Ð+Ý”—’Ð2Ñ3Ô3ð ,àÕ)Ð)Ð)¨Z¸s¼|Ð-KÐ-KÐO`Ð-Kå/<Ô/FÐGZÑ/[Ô/[ˆLÐ+Ñ,ØÐØ Ñ,µ´·²Ð@SÑ1TÔ1TÑ,õ Ð)°GÐ<Ñ<Ô<ð =Ð@PÝ!%¤Ð+;Ñ!<Ô!<�ð=ð =ð =ñ =ô =ð =ð =ð =ð =ð =ð =øøøð =ð =ð =ð =ð (×+Ò+¨G°RÑ8Ô8×<Ò<¸VÑDÔDˆJØÐ!2Ð2Ð2Ý)-¨nÑ)=Ô)=Ð&Ý $ ^°GÔ%<Ñ =Ô =�Ø)+�˜gÑ&Ø Ò&Ð&Ø.0�M (Ñ+Ø2?Ð& wÑ/Ø9;Ð& ~Ñ6Ý -Ô 6µt´zÐBXÑ7YÔ7YÑ ZÔ Z��å -Ô 7Ð8KÑ LÔ L�à-:Ô-IˆLÐ)Ñ*Ø0=Ô0EˆLÐ,Ñ-Ø3@Ô3KˆLÐ/Ñ0ð Ô'Ð3Ø3@Ô3K�Ð/Ñ0ØÔ$Ð0Ø0=Ô0E�˜_Ñ-ð !/× 2Ò 2°<Ñ @Ô @ÐØ ð Ø$×(Ò(¨°Ñ6Ô6¸*ÒDÐDØ(9¸-Ô(HÐ%Ð%Ý#Ð$5µtÑ<Ô<ð <Ø):Ð(;Ð%Ø"3ð ð �JØ!—~’~ fÑ-Ô-°Ò>Ð>ÐCYÐ]gÐCgÐCgØ%˜˜˜àDJ×DTÒDTØ&Ð'=Ô>ñEô E˜Ð%@ÑAð ˜øà"×&Ò& w°Ñ3Ô3×7Ò7¸ÀÑFÔFˆEØŒyÐ Ý˜e¥TÑ*Ô*ð 4Ý ¥¥U¨EÑ!2Ô!2Ñ3Ô3�EøØ”Ô# yÒ0Ð0Ý˜e¥TÑ*Ô*ð <¨uð <½ÀEÈ!ÄHÍtÕUZÈmÑ9\Ô9\ð <Ø;Ð;°UÐ;Ñ;Ô;�EøØ”Ô# {Ò2Ð2ØCÐCµ)¸EÑ2BÔ2BÐCÑCÔC��Ø”Ô# uÒ,Ð,°´	Ô0BÀkÒ0QÐ0QÝ˜e¥TÑ*Ô*ð qØpÐpÕ_hÐinÑ_oÔ_oÐpÑpÔp�EØ$)ˆL˜Ñ!å   g¨tÑ4Ô4ˆJØ˜>×-Ò-¨g°rÑ:Ô:Ð:Ð:À
Ð:ÈzÔObÐfkÒOkÐOkØ'¨Ô0°Ô:�ØrÐrÐkqÐrÑrÔr�Ø)/�˜XÑ&àÐà!×%Ò% lÑ3Ô3ˆ
Ø"×&Ò& }Ñ5Ô5ˆØ× Ò  Ñ)Ô)ˆØ×!Ò! (Ñ+Ô+ˆõ �j¥#Ñ&Ô&ð 	 ­2¬7¯>ª>¸*Ñ+EÔ+Eð 	 ÕJbÐcmÑJnÔJnð 	 ØHÐHÐHÐHÐHÐHà(Ð(¨Ñ4Ô4ˆIØ/8×/BÒ/BÑ/DÔ/DˆLÐ+Ñ,ØÐõ �j¥#Ñ&Ô&ð 	 ­2¬7¯>ª>¸*Ñ+EÔ+Eð 	 È*×J]ÒJ]Ð^fÑJgÔJgð 	 åŒw×Ò 
Ñ+Ô+Õ/CÒCÐCðØ#&×#BÒ#BÀ:È|Ñ#\Ô#\�LØ'Ð'øÝ ð ð ð Ý—N’NðEÀjð Eð EØðEð Eð Eñô ð ð ð ð ð ð øøøøðøøøð ×5Ò5°jÀ,ÑOÔOˆLØÐð ˆ=�Z¨
µCÑ8Ô8ˆ=½R¼W¿^º^ÈJÑ=WÔ=Wˆ=Ø$.ˆL˜Ñ!Ø  Ô)ˆEØˆ>�j¨µcÑ:Ô:ˆ>½r¼w¿~º~ÈkÑ?ZÔ?Zˆ>Ø%0ˆL˜Ñ"Ø! (Ô+ˆFð ˆ>˜cœiÐ3¸¼	Ô8JÈeÒ8SÐ8SÕXbÐchÕjnÑXoÔXoÐ8Sð	!­Xµc¬]ð 	!½tÅC¼yð 	!ð 	!ð 	!ð 	!ð 	!ð 	!ð
#ð 
#ð 
#Ðõ %(¡E¤EˆKØ*ð Rð R�Ø˜,Ð&Ð&Ø×&Ò&Ð';Ð';¸\È#Ô=NÐ<OÑ'PÔ'PÑQÔQÐQøå$ U¸ÐDÑDÔDˆFØ%+ˆL˜Ñ"ØÐs*   Â)C
Ã
CÃCÔT1 Ô1
U&Ô;!U!Õ!U&r'   r‘   rZ   c                 ó’  ‡— ddl m} d„ |                     d¦  «        pi                      ¦   «         D ¦   «         Š|                     d¦  «        pg }t	          |t
          ¦  «        r!t          |                     ¦   «         ¦  «        }t          | 	                    |¦  «        ¦  «        }‰r!t          ‰                     ¦   «         ¦  «        nd}||k    rNˆfd„t          ||dz   ¦  «        D ¦   «         }t          |¦  «        }|D ]}	|	|vr|                     |	¦  «         Œ|} |||¬¦  «        }
|
                     ¦   «         |d	<   |S )
z8Path to a vocab file and a list of extra special tokens.r   )ÚTikTokenConverterc                 óˆ   — i | ]?\  }}t          |¦  «        t          |t          ¦  «        r|d          nt          |¦  «        “Œ@S ©Úcontent)ÚintrO   rl   rS   )rC   Útoken_idrK   s      rE   rL   z<TokenizersBackend._convert_from_tiktoken.<locals>.<dictcomp>  sT   € ð 0
ð 0
ð 0
á�˜%õ �‰MŒM­z¸%ÅÑ/FÔ/FÐV˜5 Ô+Ð+ÍCÐPUÉJÌJð0
ð 0
ð 0
rG   Úadded_tokens_decoderrj   r   c                 óD   •— g | ]}‰                      |d |› d�¦  «        ‘ŒS )z<|reserved_token_z|>©rw   )rC   Úindexr¦   s     €rE   rF   z<TokenizersBackend._convert_from_tiktoken.<locals>.<listcomp>&  sE   ø€ ð ð ð àð %×(Ò(¨Ð0MÀEÐ0MÐ0MÐ0MÑNÔNðð ð rG   )r'   rj   r,   )Úconvert_slow_tokenizerr    rw   ÚitemsrO   rl   rP   ÚkeysÚlenÚload_tiktoken_bpeÚmaxÚrangerŒ   r^   rƒ   )rŽ   r'   r‘   r    rj   Úbase_vocab_sizeÚmax_added_token_idÚ
new_extrasÚset_new_extrasÚspecial_tokenrš   r¦   s              @rE   r‹   z(TokenizersBackend._convert_from_tiktoken  s£  ø€ ð 	>Ð=Ð=Ð=Ð=Ð=ð0
ð 0
à$0×$4Ò$4Ð5KÑ$LÔ$LÐ$RÐPR×#YÒ#YÑ#[Ô#[ð0
ñ 0
ô 0
Ðð
 +7×*:Ò*:Ð;QÑ*RÔ*RÐ*XÐVXÐÝÐ*­DÑ1Ô1ð 	EÝ#'Ð(<×(AÒ(AÑ(CÔ(CÑ#DÔ#DÐ õ Ð/×AÒAÀ*ÑMÔMÑNÔNˆð BVÐ\�SÐ!5×!:Ò!:Ñ!<Ô!<Ñ=Ô=Ð=Ð[\ÐØ Ò0Ð0ðð ð ð å" ?Ð4FÈÑ4JÑKÔKðñ ô ˆJõ ! ™_œ_ˆNØ!5ð 5ð 5�Ø ¨Ð6Ð6Ø×%Ò% mÑ4Ô4Ð4øØ#-Ð à%Ð%°ÐRfÐgÑgÔgˆ	Ø+4×+>Ò+>Ñ+@Ô+@ˆÐ'Ñ(ØÐrG   c                 ó¸  — ddl m}  ||¦  «        } |j        | j        fi |¤Ž}	 ddl m} |                     | j        ¦  «        }|�t          |d¦  «        r |j        di |¤Ž}n=# t          $ r0}t                               d| j        › d|› d�¦  «         Y d }~nd }~ww xY wt          | d¦  «        r | j        di |¤Ž}d	|v�r•| t          u s
d
| j        v�r‚|                     dd ¦  «        }|                     dd ¦  «        }	|                     d¦  «        pi }
|�–|
r”d„ |                     ¦   «         D ¦   «         }|
                     ¦   «         D ]a\  }}t#          |¦  «        }t%          |¦  «        }|                     |¦  «        }|r'||k    r!||vr|                     |¦  «        ||<   |||<   Œbt'          j        |j        ||	¬¦  «        }|�‰||d	<   |j        j        }|j        dk    r|                     d|j        pd¦  «         |j        dk    r|                     d|j        pd¦  «         |j        dk    r|                     d|j        pd¦  «         |S )Nr   )ÚSentencePieceExtractor)ÚSLOW_TO_FAST_CONVERTERSÚconvert_from_spmz,Could not reorder vocab using converter for z due to z/. Falling back to raw SentencePiece extraction.Úconvert_from_spm_modelr,   r+   r2   r3   r¦   c                 ó   — i | ]\  }}||“Œ	S rA   rA   )rC   rK   r¥   s      rE   rL   zATokenizersBackend._convert_from_sentencepiece.<locals>.<dictcomp>T  s   € ÐTÐTÐT±?°5¸(˜x¨ÐTÐTÐTrG   )Úprotor2   r3   r   rd   ú<s>re   ú</s>rc   z<unk>rA   )rª   r·   Úextractr0   r¸   rw   r   Úhasattrr¹   rˆ   r‰   rŠ   rº   r)   rq   rm   r«   r¤   rS   r   Úbuild_tokenizer_from_spm_protor¼   Útrainer_specÚbos_idÚ
setdefaultÚ	bos_pieceÚeos_idÚ	eos_pieceÚunk_idÚ	unk_piece)rŽ   r'   r‘   r·   Ú	extractorr¸   Úconverter_classr›   r2   r3   r¦   Úid_to_tokenr¥   Ú	new_tokenÚcurrent_tokenr,   Ú
proto_specs                    rE   r‡   z-TokenizersBackend._convert_from_sentencepiece4  s,  € àBÐBÐBÐBÐBÐBð +Ð*¨:Ñ6Ô6ˆ	Ø(�yÔ(¨¬ÐCÐC°lÐCÐCˆð		ØGÐGÐGÐGÐGÐGà5×9Ò9¸#¼,ÑGÔGˆOØÐ*­w°ÐHZÑ/[Ô/[Ð*Ø?˜Ô?ÐOÐOÀ,ÐOÐO�øøÝð 	ð 	ð 	Ý�NŠNð H¸s¼|ð  Hð  HÐUVð  Hð  Hð  Hñô ð ð ð ð ð ð øøøøð	øøøõ �3Ð0Ñ1Ô1ð 	FØ5˜3Ô5ÐEÐE¸ÐEÐEˆLð
  \Ð1Ñ1°sÕ>OÐ7OÐ7OÐS]ÐehÔeqÐSqÑSqØ ×$Ò$ W¨dÑ3Ô3ˆEØ!×%Ò% h°Ñ5Ô5ˆFð $0×#3Ò#3Ð4JÑ#KÔ#KÐ#QÈrÐ ØÐ Ð%9Ð ØTÐTÀeÇkÂkÁmÄmÐTÑTÔT�Ø+?×+EÒ+EÑ+GÔ+Gð :ð :Ñ'�H˜iÝ" 8™}œ}�HÝ # I¡¤�IØ$/§O¢O°HÑ$=Ô$=�MØ$ð :¨¸)Ò)CÐ)CÈ	ÐY^ÐH^ÐH^Ø+0¯9ª9°]Ñ+CÔ+C˜˜iÑ(Ø09˜ HÑ-øå+ÔJØ”oØØð ñ  ô  Ðð
  Ð+Ø3C�Ð/Ñ0ð 'œ_Ô9�
ØÔ$¨Ò)Ð)Ø ×+Ò+¨K¸Ô9MÐ9VÐQVÑWÔWÐWØÔ$¨Ò)Ð)Ø ×+Ò+¨K¸Ô9MÐ9WÐQWÑXÔXÐXØÔ$¨Ò)Ð)Ø ×+Ò+¨K¸Ô9MÐ9XÐQXÑYÔYÐYØÐs   ¦?A& Á&
B Á0&BÂB c           	      óº  •‡%— |                      dd ¦  «        }|                      dd ¦  «        }|                      dd ¦  «         |                      dd ¦  «        }|                      dd ¦  «        }|                      dd ¦  «        }|                     di ¦  «        }|                     dd	¦  «        }	|                     d
¦  «        }
|                     d¦  «        }|                     d¦  «        }d }|�t          j        |¦  «        }�n |�5t          j                             |¦  «        rt          j        |¦  «        }�né|�Óddl	m
}m}m} t          |                     dd¦  «        |fi |¤Ž} ||¦  «        \  }}|d         |v r ||¦  «        \  }}}n-t          |¦  «        }|d         d         }|d         }|d         }t          ||¦  «        \  }}|                     |¦  «         t#          |¦  «        dk    r|                     |¦  «         �n| j        €÷|�õ|�Qt'          |t(          ¦  «        r|nd„ t+          |¦  «        D ¦   «         }t          t-          ||dd ¬¦  «        ¦  «        }n¸t'          |t(          ¦  «        r!t          t-          |g dd ¬¦  «        ¦  «        }n‚t'          |t.          ¦  «        rV|rTt'          |d         t0          t.          f¦  «        r2t          t3          ||                     dd¦  «        ¬¦  «        ¦  «        }n| j        €t5          d¦  «        ‚|€5|€3| j        €,|                     dd¦  «         |                     dd ¦  «         |�|| _        | j        €t5          d!¦  «        ‚|                      d"d ¦  «        p| j        j        p|}|�ƒ | j        j        d@i |¤Ž |                     d#|d#         ¦  «         |                     d$|d%         ¦  «         |                     d&|d&         ¦  «         |                     d'|d(         ¦  «         n| j                             ¦   «          |                      d)d ¦  «        p| j        j        p|}|�ž | j        j         d@i |¤Ž |                     d*|d*         ¦  «         |                     d+|d,         ¦  «         |                     d-|d%         ¦  «         |                     d#|d.         ¦  «         |                     d/|d/         ¦  «         d0|vrd1|d0<   d2|v pd3|v }|                     d2d	¦  «        | _!        |                     d3d	¦  «        | _"        |                      d4d ¦  «        x}r|| j        _#        |p| j        j#        d u | _$         tK          ¦   «         j&        d@i |¤Ž |
�|
| _'        |	| _(        | j)        | j        _*        d5„ | j+        D ¦   «         Š%ˆ%fd6„tY          | -                    ¦   «         d7„ ¬8¦  «        D ¦   «         }t/          | j.         /                    ¦   «         ¦  «        d9„ |D ¦   «         z   }| j0         1                    ¦   «         D ]/} | €Œte          | ¦  «        |vr| |vr| 3                    | ¦  «         Œ0| j4        D ],}!te          |!¦  «        |vr|!|vr| 3                    |!¦  «         Œ-t#          |¦  «        dk    r±g }"d:„ | j0         1                    ¦   «         D ¦   «         }#|D ]r}!t'          |!td          ¦  «        rtk          |!d¬;¦  «        }!n4t'          |!tj          ¦  «        r|!j6        ste          |!¦  «        |#v rd|!_6        |" 3                    |!¦  «         Œs|"r|  7                    |"¦  «         	 | j         8                    ¦   «         }$n# tr          $ r d}$Y nw xY w|$d<k    rztu          | j        d=d ¦  «        �d|                      dd ¦  «          | j;        | j        | j<                             dd ¦  «        f| j<        |                      d>d ¦  «        d?œ|¤Ž| _        | j$        p| j        j#        d u | _$        | j$        r|  =                    ¦   «          d S d S )ANr8   r9   r?   r,   Ú	gguf_filer&   r¦   Úadd_prefix_spaceFr'   r2   r3   r   )ÚGGUF_TOKENIZER_ARCHSÚget_gguf_tokenizerÚread_gguf_metadataÚname_or_pathÚ zgeneral.architectureÚconfigr•   Ú	tokenizerÚtokenizer_configr   c                 ó    — i | ]\  }\  }}||“ŒS rA   rA   )rC   rJ   ÚwÚ_s       rE   rL   z.TokenizersBackend.__init__.<locals>.<dictcomp>   s%   € ÐCkÐCkÐCkÉYÈQÑPVÐQRÐTUÀAÀqÐCkÐCkÐCkrG   T)r2   r3   Úfuse_unkÚdropoutrÈ   )r2   rÈ   a9  Couldn't instantiate the backend tokenizer from one of: 
(1) a `tokenizers` library serialization file, 
(2) a slow tokenizer instance to convert or 
(3) an equivalent slow tokenizer class to instantiate and convert. 
You need to have sentencepiece or tiktoken installed to convert a slow tokenizer to a fast one.rd   r½   re   r¾   z3The backend tokenizer is not correctly initialized.r7   Ú
max_lengthÚtruncation_sideÚ	directionÚstrideÚtruncation_strategyÚstrategyr6   rb   Úpad_token_type_idÚpad_type_idÚpadding_sideÚlengthÚpad_to_multiple_ofÚbackendÚ
tokenizersÚadd_bos_tokenÚadd_eos_tokenr5   c                 óF   — h | ]}t          t          |¦  «        ¦  «        ’ŒS rA   ©ÚhashÚrepr©rC   rK   s     rE   ú	<setcomp>z-TokenizersBackend.__init__.<locals>.<setcomp>ß  s&   € Ð$^Ð$^Ð$^¸5¥T­$¨u©+¬+Ñ%6Ô%6Ð$^Ð$^Ð$^rG   c                 óV   •— g | ]%\  }}t          t          |¦  «        ¦  «        ‰v¯#|‘Œ&S rA   rð   )rC   r©   rK   Úadded_tokens_decoder_hashs      €rE   rF   z.TokenizersBackend.__init__.<locals>.<listcomp>à  sA   ø€ ð 
ð 
ð 
á��uÝ•D˜‘K”KÑ Ô Ð(AÐAÐAð àAÐAÐArG   c                 ó   — | d         S ©Nr   rA   )Úxs    rE   ú<lambda>z,TokenizersBackend.__init__.<locals>.<lambda>â  s   € ÐSTÐUVÔSW€ rG   ©r�   c                 ó,   — g | ]}t          |¦  «        ‘ŒS rA   ©rS   ró   s     rE   rF   z.TokenizersBackend.__init__.<locals>.<listcomp>å  s   € Ð;bÐ;bÐ;bÈ5½CÀ¹J¼JÐ;bÐ;bÐ;brG   c                 ó0   — g | ]}|¯t          |¦  «        ‘ŒS rA   rý   )rC   Úts     rE   rF   z.TokenizersBackend.__init__.<locals>.<listcomp>ö  s$   € ÐWÐWÐW¨1ÐUVÐW¥ A¡¤ÐWÐWÐWrG   )Úspeciali † Úpre_tokenizerÚfix_mistral_regex)Úinit_kwargsr  rA   )>rm   rw   ÚcopyÚdeepcopyrn   ro   rp   rr   rs   Úintegrations.ggufrÓ   rÔ   rÕ   r   r   r   r�   r­   Ú
_tokenizerrO   rl   r€   r   rP   rB   r   Ú
ValueErrorrÄ   r{   Úenable_truncationÚno_truncationrz   Úenable_paddingÚ_add_bos_tokenÚ_add_eos_tokenr5   Ú_should_update_post_processorÚsuperr+   r'   rÒ   Úsplit_special_tokensÚencode_special_tokensr¦   Úsortedr«   Úadded_tokens_encoderr¬   Ú_special_tokens_maprY   rS   r^   Ú_extra_special_tokensr   r   Ú
add_tokensÚget_vocab_sizeÚNotImplementedErrorr�   Ú_patch_mistral_regexr  Úupdate_post_processor)'ÚselfÚargsr�   r8   r9   r,   rÑ   r’   r¦   rÒ   r'   r2   r3   Úfast_tokenizerrÓ   rÔ   rÕ   Ú	gguf_pathÚmetadatarÝ   ÚarchitectureÚtokenizer_dictrÚ   Ú
gguf_paramÚadditional_kwargsÚ
vocab_dictÚ_truncationÚ_paddingÚexplicit_bos_eos_in_kwargsr5   Útokens_to_addÚencoderÚspecial_token_valuerK   ÚtokensÚall_named_tokensÚ
vocab_sizerö   Ú	__class__s'                                        @€rE   r+   zTokenizersBackend.__init__p  sM	  øø€ ð "Ÿ:š:Ð&8¸$Ñ?Ô?ÐØŸ
š
 ?°DÑ9Ô9ˆð 	�
Š
Ð.°Ñ5Ô5Ð5à!Ÿ:š:Ð&8¸$Ñ?Ô?ÐØ—J’J˜{¨DÑ1Ô1ˆ	Ø$ŸjšjÐ)9¸4Ñ@Ô@Ðà%ŸzšzÐ*@À"ÑEÔEÐà!Ÿ:š:Ð&8¸%Ñ@Ô@ÐØ—Z’Z Ñ-Ô-ˆ
à—
’
˜7Ñ#Ô#ˆØ—’˜HÑ%Ô%ˆàˆØÐ'Ý!œ]Ð+;Ñ<Ô<ˆN‰NØ Ð,µ´·²Ð@SÑ1TÔ1TÐ,å*Ô4Ð5HÑIÔIˆN‰NØÐ"àgÐgÐgÐgÐgÐgÐgÐgÐgÐgå# F§J¢J¨~¸rÑ$BÔ$BÀIÐXÐXÐQWÐXÐXˆIà,Ð,¨YÑ7Ô7‰KˆH�aØÐ.Ô/Ð3GÐGÐGØASÐASÐT]ÑA^ÔA^Ñ>�˜nÐ.>Ð.>å1°)Ñ<Ô<�
Ø)¨(Ô3°LÔA�Ø!+¨KÔ!8�Ø#-Ð.@Ô#AÐ Ý0FÀ|ÐUcÑ0dÔ0dÑ-ˆNÐ-Ø�MŠMÐ*Ñ+Ô+Ð+ÝÐ$Ñ%Ô%¨Ò)Ð)Ø—’Ð/Ñ0Ô0Ð0ùØŒ_Ð$¨Ð):àÐ!Ý&0°½Ñ&=Ô&=Ðk˜U˜UÐCkÐCkÕZcÐdiÑZjÔZjÐCkÑCkÔCk�
Ý!.­s¸ÈFÐ]aÐkoÐ/pÑ/pÔ/pÑ!qÔ!q��Ý˜E¥4Ñ(Ô(ð eÝ!.­s¸ÀrÐTXÐbfÐ/gÑ/gÔ/gÑ!hÔ!h��Ý˜E¥4Ñ(Ô(ð e¨Uð eµzÀ%ÈÄ(ÍUÕTXÈMÑ7ZÔ7Zð eÝ!.­w¸UÈ6Ï:Ê:ÐV^Ð`aÑKbÔKbÐ/cÑ/cÔ/cÑ!dÔ!d�øØŒ_Ð$Ýðrñô ð ð Ð&Ð+;Ð+CÈÌÐH_Ø×Ò˜k¨5Ñ1Ô1Ð1Ø×Ò˜k¨6Ñ2Ô2Ð2àÐ%Ø,ˆDŒOàŒ?Ð"ÝÐRÑSÔSÐSà—j’jÐ!7¸Ñ>Ô>ÐpÀ$Ä/ÔB\ÐpÐ`pˆØÐ"Ø-ˆDŒOÔ-Ð<Ð<°Ð<Ð<Ð<Ø×Ò˜l¨K¸Ô,EÑFÔFÐFØ×ÒÐ/°¸[Ô1IÑJÔJÐJØ×Ò˜h¨°HÔ(=Ñ>Ô>Ð>Ø×ÒÐ3°[ÀÔ5LÑMÔMÐMÐMàŒO×)Ò)Ñ+Ô+Ð+à—:’:Ð1°4Ñ8Ô8Ðd¸D¼OÔ<SÐdÐWdˆØÐØ*ˆDŒOÔ*Ð6Ð6¨XÐ6Ð6Ð6Ø×Ò˜k¨8°KÔ+@ÑAÔAÐAØ×ÒÐ1°8¸MÔ3JÑKÔKÐKØ×Ò˜n¨h°{Ô.CÑDÔDÐDØ×Ò˜l¨H°XÔ,>Ñ?Ô?Ð?Ø×ÒÐ2°HÐ=QÔ4RÑSÔSÐSð ˜FÐ"Ð"Ø ,ˆF�9Ñà%4¸Ð%>Ð%[À/ÐU[ÐB[Ð"Ø$Ÿjšj¨¸%Ñ@Ô@ˆÔØ$Ÿjšj¨¸%Ñ@Ô@ˆÔØ#ŸZšZÐ(8¸$Ñ?Ô?Ð?ˆ>ð 	<Ø-;ˆDŒOÔ*Ø-GÐ-qÈ4Ì?ÔKiÐmqÐKqˆÔ*à�‰ŒÔÐ"Ð"˜6Ð"Ð"Ð"àÐ!Ø(ˆDŒOà 0ˆÔØ04Ô0IˆŒÔ-à$^Ð$^ÀDÔD]Ð$^Ñ$^Ô$^Ð!ð
ð 
ð 
ð 
å &Ð';×'AÒ'AÑ'CÔ'CÈÈÐ XÑ XÔ Xð
ñ 
ô 
ˆõ
 �tÔ0×5Ò5Ñ7Ô7Ñ8Ô8Ð;bÐ;bÐTaÐ;bÑ;bÔ;bÑbˆð $(Ô#;×#BÒ#BÑ#DÔ#Dð 	:ð 	:ÐØ"Ð*ØÝÐ&Ñ'Ô'¨wÐ6Ð6Ð;NÐVcÐ;cÐ;cØ×$Ò$Ð%8Ñ9Ô9Ð9øð Ô/ð 	,ð 	,ˆEÝ�5‰zŒz Ð(Ð(¨U¸-Ð-GÐ-GØ×$Ò$ UÑ+Ô+Ð+øåˆ}ÑÔ Ò!Ð!ØˆFØWÐW°Ô0H×0OÒ0OÑ0QÔ0QÐWÑWÔWÐØ&ð %ð %�Ý˜e¥SÑ)Ô)ð -å& u°dÐ;Ñ;Ô;�E�EÝ ¥zÑ2Ô2ð -à œ=ð -­S°©Z¬ZÐ;KÐ-KÐ-KØ(,˜œØ—’˜eÑ$Ô$Ð$Ð$Øð (à—’ Ñ'Ô'Ð'ð	Øœ×7Ò7Ñ9Ô9ˆJˆJøÝ"ð 	ð 	ð 	ØˆJˆJˆJð	øøøð ˜ÒÐ¥7¨4¬?¸OÈTÑ#RÔ#RÐ#^Ø�JŠJ�{ DÑ)Ô)Ð)Ø7˜dÔ7Ø”ØÔ ×$Ò$ ^°TÑ:Ô:ðð !Ô,Ø"(§*¢*Ð-@À$Ñ"GÔ"Gð	ð ð
 ðð ˆDŒOð Ô.ÐX°$´/Ô2PÐTXÐ2Xð 	Ô*ð Ô-ð 	)Ø×&Ò&Ñ(Ô(Ð(Ð(Ð(ð	)ð 	)s   Ý8^ Þ^!Þ ^!c                 ó   — dS )NTrA   ©r  s    rE   Úis_fastzTokenizersBackend.is_fast  s   € àˆtrG   c                 óÒ   — d| j         v r]| j         d                              d¦  «        r=t          | d¦  «        r+| j        r$t          j                             | j        ¦  «        S dS dS )zÌ
        `bool`: Whether or not the slow tokenizer can be saved. For a sentencepiece based slow tokenizer, this
        can only be `True` if the original `"sentencepiece.model"` was not deleted.
        r'   rX   FT)Úvocab_files_namesr„   rÀ   r'   rn   ro   rp   r0  s    rE   Úcan_save_slow_tokenizerz)TokenizersBackend.can_save_slow_tokenizer  si   € ð ˜4Ô1Ð1Ð1°dÔ6LÈ\Ô6Z×6cÒ6cÐdlÑ6mÔ6mÐ1Ý�t˜\Ñ*Ô*ð 7¨t¬ð 7å”w—~’~ d¤oÑ6Ô6Ð6Ø�5à�4rG   Úsave_directoryÚfilename_prefixc                 ó¤  — t           j                             |¦  «        s t                               d|› d�¦  «         d S t           j                             ||r|dz   ndt          d         z   ¦  «        }t           j                             | j        ¦  «        t           j                             |¦  «        k    rt          | j        |¦  «         |fS )NzVocabulary path (z) should be a directoryú-r×   r'   )
rn   ro   Úisdirr‰   ÚerrorÚjoinÚVOCAB_FILES_NAMESÚabspathr'   r   )r  r5  r6  Úout_vocab_files       rE   Úsave_vocabularyz!TokenizersBackend.save_vocabulary,  s¶   € ÝŒw�}Š}˜^Ñ,Ô,ð 	Ý�LŠLÐT¨^ÐTÐTÐTÑUÔUÐUØˆFÝœŸšØ°oÐM˜_¨sÑ2Ð2È2ÕQbÐcoÔQpÑpñ
ô 
ˆõ Œ7�?Š?˜4œ?Ñ+Ô+­r¬w¯ª¸~Ñ/NÔ/NÒNÐNÝ�T”_ nÑ5Ô5Ð5àÐ Ð rG   Úlegacy_formatÚpush_to_hubÚsave_format.c           	      ót  •— |dvrt          d|›d�¦  «        ‚|dk    rýddlm} i }|r4t          j                             |¦  «        r|                      |¦  «        } || |¦  «        }	|r¯|                     dd¦  «        }
|                     d	t          |¦  «         	                    t          j        j
        ¦  «        d
         ¦  «        } t          ¦   «         j        |fddi|¤Žj        }|                      ||||
|                     d¦  «        ¬¦  «         |	S  t!          ¦   «         j        |f|||dœ|¤ŽS )aþ  
        Save the full tokenizer state.


        Extends [`~tokenization_utils_base.PreTrainedTokenizerBase.save_pretrained`] with
        `save_format` support, for tokenizers converted from a native vocabulary file
        (e.g. Mistral's `tekken.json`). See the base method for `legacy_format`,
        `filename_prefix`, and `push_to_hub`.

        Args:
            save_format (`str`, *optional*):
                `"mistral"` to save as a native `tekken.json` by copying the original
                file (requires the original tekken vocabulary file to be available). The name is
                normalized to `tekken.json` on save, and the instantiated tokenizer must still match
                it exactly.
                `"hf"` or `None` for the default HuggingFace format.
            kwargs (`dict[str, Any]`, *optional*):
                Additional key word arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method.

        Returns:
            A tuple of `str`: The files saved.
        )NÚhfÚmistralzUnknown save_format=z$. Supported values: 'hf', 'mistral'.rE  r   )Úsave_tekken_formatÚcommit_messageNÚrepo_idéÿÿÿÿÚexist_okTrK   )Úworking_dirrH  Úfiles_timestampsrG  rK   )r@  r6  rA  )r  r‚   rF  rn   ro   r9  Ú_get_files_timestampsrm   rS   rT   Úsepr"   Úcreate_reporH  Ú_upload_modified_filesrw   r  Úsave_pretrained)r  r5  r@  r6  rA  rB  r�   rF  rL  Ú
save_filesrG  rH  r.  s               €rE   rQ  z!TokenizersBackend.save_pretrained9  s…  ø€ ð> Ð5Ð5Ð5ÝÐg°KÐgÐgÐgÑhÔhÐhà˜)Ò#Ð#ØJÐJÐJÐJÐJÐJð
  "ÐØð N�rœwŸ}š}¨^Ñ<Ô<ð NØ#'×#=Ò#=¸nÑ#MÔ#MÐ à+Ð+¨D°.ÑAÔAˆJàð 
Ø!'§¢Ð,<¸dÑ!CÔ!C�Ø Ÿ*š* Yµ°NÑ0CÔ0C×0IÒ0IÍ"Ì'Ì+Ñ0VÔ0VÐWYÔ0ZÑ[Ô[�Ø.�&™(œ(Ô.¨wÐPÐPÀÐPÈÐPÐPÔX�Ø×+Ò+Ø .Ø#Ø%5Ø#1Ø Ÿ*š* WÑ-Ô-ð ,ñ ô ð ð Ðà&�u‰wŒwÔ&Øð
à'Ø+Ø#ð	
ð 
ð
 ð
ð 
ð 	
rG   c                 óÎ  — | j         }| j        }|€| j        rd| _        | j        }| j        }|€| j        rd| _        | j        r|dz   nd› d| j        rd|z   dz   nd› �}|› | j        rd|z   dz   nd› d	| j        rd|z   dz   nd› �}g }| j        r|                     ||f¦  «         | j        r|                     ||f¦  «         t          j        |||¬
¦  «        | j	        _
        dS )ze
        Updates the underlying post processor with the current `bos_token` and `eos_token`.
        NFz:0 r×   z$A:0rR   z:0z:1z $B:1)ÚsingleÚpairÚspecial_tokens)rd   Úbos_token_idrí   re   Úeos_token_idrî   r^   r	   ÚTemplateProcessingr  r5   )r  ÚbosrW  ÚeosrX  rT  rU  rV  s           rE   r  z'TokenizersBackend.update_post_processor}  s]  € ð ŒnˆØÔ(ˆØˆ;˜4Ô-ˆ;Ø!&ˆDÔàŒnˆØÔ(ˆØˆ;˜4Ô-ˆ;Ø!&ˆDÔà%)Ô%7Ð?�S˜5‘[�[¸RÐwÐwÐ[_Ô[mÐEuÀcÈCÁiÐRVÑFVÐFVÐsuÐwÐwˆØð  D°Ô0BÐJ˜3 ™9 tÑ+Ð+Èð  Dð  DÐgkÔgyð  RBÐRUÐX[ÑR[Ð^bÑRbÐRbð  @Bð  Dð  DˆàˆØÔð 	7Ø×!Ò! 3¨Ð"5Ñ6Ô6Ð6ØÔð 	7Ø×!Ò! 3¨Ð"5Ñ6Ô6Ð6Ý)3Ô)FØ °^ð*
ñ *
ô *
ˆŒÔ&Ð&Ð&rG   c                 ó$   — t          | dd¦  «        S )Nr  F©r�   r0  s    rE   rî   zTokenizersBackend.add_eos_token—  ó   € å�tÐ-¨uÑ5Ô5Ð5rG   c                 ó$   — t          | dd¦  «        S )Nr  Fr]  r0  s    rE   rí   zTokenizersBackend.add_bos_token›  r^  rG   c                 óf   — t                                | d|¦  «         |                      ¦   «          d S )Nr  ©ÚobjectÚ__setattr__r  ©r  Úvalues     rE   rî   zTokenizersBackend.add_eos_tokenŸ  ó3   € å×Ò˜4Ð!1°5Ñ9Ô9Ð9Ø×"Ò"Ñ$Ô$Ð$Ð$Ð$rG   c                 óf   — t                                | d|¦  «         |                      ¦   «          d S )Nr  ra  rd  s     rE   rí   zTokenizersBackend.add_bos_token¤  rf  rG   c                 óˆ  — g }| j                              ¦   «         D ]j}|€Œt          |t          ¦  «        r|                     |¦  «         Œ0t          |t
          ¦  «        r%|                     t          |dd¬¦  «        ¦  «         Œk| j        D ]g}t          |t          ¦  «        r|                     |¦  «         Œ-t          |t
          ¦  «        r%|                     t          |dd¬¦  «        ¦  «         Œh|r|                      |d¬¦  «         t          | dd¦  «        s| j	        j
        €|                      ¦   «          dS dS )a[  
        Post-initialization hook that runs after the tokenizer is fully set up.
        This is called by from_pretrained() after loading the tokenizer, which allows
        us to add any special tokens that may have been passed as AddedToken objects.

        Child classes should call super()._post_init() if they override this method.
        NTF)r   Ú
normalized)rV  r  )r  rY   rO   r   r^   rS   r  r  r�   r  r5   r  )r  r(  Útoken_valuerK   s       rE   Ú
_post_initzTokenizersBackend._post_init©  sl  € ð ˆàÔ3×:Ò:Ñ<Ô<ð 	^ð 	^ˆKØÐ"ØÝ˜+¥zÑ2Ô2ð ^Ø×$Ò$ [Ñ1Ô1Ð1Ð1Ý˜K­Ñ-Ô-ð ^Ø×$Ò$¥Z°ÀTÐV[Ð%\Ñ%\Ô%\Ñ]Ô]Ð]øð Ô/ð 	Xð 	XˆEÝ˜%¥Ñ,Ô,ð XØ×$Ò$ UÑ+Ô+Ð+Ð+Ý˜E¥3Ñ'Ô'ð XØ×$Ò$¥Z°¸tÐPUÐ%VÑ%VÔ%VÑWÔWÐWøàð 	@à�OŠO˜M¸$ˆOÑ?Ô?Ð?å�4Ð8¸$Ñ?Ô?ð 	)À4Ä?ÔCaÐCiØ×&Ò&Ñ(Ô(Ð(Ð(Ð(ð DjÐCirG   c                 ó8   — | j                              d¬¦  «        S )zP
        `int`: Size of the base vocabulary (without the added tokens).
        F©Úwith_added_tokens©r  r  r0  s    rE   r-  zTokenizersBackend.vocab_sizeÉ  s   € ð
 Œ×-Ò-ÀÐ-ÑFÔFÐFrG   c                 ó8   — | j                              d¬¦  «        S )NTrm  )r  Ú	get_vocabr0  s    rE   rq  zTokenizersBackend.get_vocabÐ  s   € ØŒ×(Ò(¸4Ð(Ñ@Ô@Ð@rG   c                 ó*   — |                       ¦   «         S r\   )rq  r0  s    rE   r2   zTokenizersBackend.vocabÓ  s   € à�~Š~ÑÔÐrG   c                 óh   — d„ t          | j                             ¦   «         d„ ¬¦  «        D ¦   «         S )zÃ
        Returns the sorted mapping from string to index. The added tokens encoder is cached for performance
        optimisation in `self._added_tokens_encoder` for the slow tokenizers.
        c                 ó$   — i | ]\  }}|j         |“ŒS rA   r¢   ©rC   ÚvÚks      rE   rL   z:TokenizersBackend.added_tokens_encoder.<locals>.<dictcomp>Ý  ó    € ÐmÐmÐm¡  A�”	˜1ÐmÐmÐmrG   c                 ó   — | d         S rø   rA   ©rD   s    rE   rú   z8TokenizersBackend.added_tokens_encoder.<locals>.<lambda>Ý  ó   € ÐdhÐijÔdk€ rG   rû   ©r  r¦   r«   r0  s    rE   r  z&TokenizersBackend.added_tokens_encoder×  s9   € ð nÐm­°Ô0I×0OÒ0OÑ0QÔ0QÐWkÐWkÐ)lÑ)lÔ)lÐmÑmÔmÐmrG   c                 ó4   — | j                              ¦   «         S )z¦
        Returns the added tokens in the vocabulary as a dictionary of index to AddedToken.

        Returns:
            `dict[str, int]`: The added tokens.
        )r  Úget_added_tokens_decoderr0  s    rE   r¦   z&TokenizersBackend.added_tokens_decoderß  s   € ð Œ×7Ò7Ñ9Ô9Ð9rG   c                 óh   — d„ t          | j                             ¦   «         d„ ¬¦  «        D ¦   «         S )z¡
        Returns the added tokens in the vocabulary as a dictionary of token to index.

        Returns:
            `dict[str, int]`: The added tokens.
        c                 ó$   — i | ]\  }}|j         |“ŒS rA   r¢   ru  s      rE   rL   z5TokenizersBackend.get_added_vocab.<locals>.<dictcomp>õ  rx  rG   c                 ó   — | d         S rø   rA   rz  s    rE   rú   z3TokenizersBackend.get_added_vocab.<locals>.<lambda>õ  r{  rG   rû   r|  r0  s    rE   Úget_added_vocabz!TokenizersBackend.get_added_vocabî  s9   € ð nÐm­°Ô0I×0OÒ0OÑ0QÔ0QÐWkÐWkÐ)lÑ)lÔ)lÐmÑmÔmÐmrG   c                 ó   — dS )zN
        Returns True, to avoid expensive `assert tokenizer` gotchas.
        TrA   r0  s    rE   Ú__bool__zTokenizersBackend.__bool__÷  s	   € ð ˆtrG   c                 ó8   — | j                              d¬¦  «        S )zD
        Size of the full vocabulary with the added tokens.
        Trm  ro  r0  s    rE   Ú__len__zTokenizersBackend.__len__ý  s   € ð Œ×-Ò-ÀÐ-ÑEÔEÐErG   c                 ó   — | j         S )zc
        `tokenizers.implementations.BaseTokenizer`: The Rust tokenizer used as a backend.
        )r  r0  s    rE   Úbackend_tokenizerz#TokenizersBackend.backend_tokenizer  s   € ð
 ŒÐrG   c                 ó   — | j         j        S )zU
        `tokenizers.decoders.Decoder`: The Rust decoder for this tokenizer.
        )r  Údecoderr0  s    rE   rŠ  zTokenizersBackend.decoder
  s   € ð
 ŒÔ&Ð&rG   Tr/   Úreturn_token_type_idsÚreturn_attention_maskÚreturn_overflowing_tokensÚreturn_special_tokens_maskÚreturn_offsets_mappingÚreturn_lengthÚverbosec	                 óF  — |€	d| j         v }|€	d| j         v }|r|j        �|g|j        z   }	n|g}	t          t          ¦  «        }
|	D ]Ù}|
d                              |j        ¦  «         |r |
d                              |j        ¦  «         |r |
d                              |j        ¦  «         |r |
d                              |j        ¦  «         |r |
d                              |j	        ¦  «         |r-|
d                              t          |j        ¦  «        ¦  «         ŒÚ|
|	fS )a¢  
        Convert the encoding representation (from low-level HuggingFace tokenizer output) to a python Dict and a list
        of encodings, take care of building a batch from overflowing tokens.

        Overflowing tokens are converted to additional examples (like batches) so the output values of the dict are
        lists (overflows) of lists (tokens).

        Output shape: (overflows, sequence length)
        NÚtoken_type_idsÚattention_maskÚ	input_idsÚspecial_tokens_maskÚoffset_mappingré   )Úmodel_input_namesÚoverflowingr   rP   r^   ÚidsÚtype_idsr”  r–  Úoffsetsr­   )r  r/   r‹  rŒ  r�  rŽ  r�  r�  r‘  Ú	encodingsÚencoding_dictr›   s               rE   Ú_convert_encodingz#TokenizersBackend._convert_encoding  sV  € ð( !Ð(Ø$4¸Ô8NÐ$NÐ!Ø Ð(Ø$4¸Ô8NÐ$NÐ!à$ð 	#¨Ô)=Ð)IØ!˜
 XÔ%9Ñ9ˆIˆIà!˜
ˆIå#¥DÑ)Ô)ˆØð 	;ð 	;ˆAØ˜+Ô&×-Ò-¨a¬eÑ4Ô4Ð4à$ð CØÐ.Ô/×6Ò6°q´zÑBÔBÐBØ$ð IØÐ.Ô/×6Ò6°qÔ7GÑHÔHÐHØ)ð SØÐ3Ô4×;Ò;¸AÔ<QÑRÔRÐRØ%ð BØÐ.Ô/×6Ò6°q´yÑAÔAÐAØð ;Ø˜hÔ'×.Ò.­s°1´5©z¬zÑ:Ô:Ð:øà˜iÐ'Ð'rG   rK   c                 óL   — | j                              |¦  «        }|€| j        S |S r\   )r  Útoken_to_idÚunk_token_id)r  rK   r©   s      rE   Ú#_convert_token_to_id_with_added_vocz5TokenizersBackend._convert_token_to_id_with_added_voc@  s*   € Ø”×+Ò+¨EÑ2Ô2ˆØˆ=ØÔ$Ð$ØˆrG   r©   c                 óP   — | j                              t          |¦  «        ¦  «        S r\   )r  rÌ   r¤   )r  r©   s     rE   Ú_convert_id_to_tokenz&TokenizersBackend._convert_id_to_tokenF  s   € ØŒ×*Ò*­3¨u©:¬:Ñ6Ô6Ð6rG   Ú
new_tokensc                 ón   — |r| j                              |¦  «        S | j                              |¦  «        S r\   )r  Úadd_special_tokensr  )r  r¦  rV  s      rE   Ú_add_tokenszTokenizersBackend._add_tokensI  s7   € Øð 	BØ”?×5Ò5°jÑAÔAÐAàŒ×)Ò)¨*Ñ5Ô5Ð5rG   rU  c                 ó6   — | j                              |¦  «        S )aG  
        Returns the number of added tokens when encoding a sequence with special tokens.

        <Tip>

        This encodes a dummy input and checks the number of added tokens, and is therefore not efficient. Do not put
        this inside your training loop.

        </Tip>

        Args:
            pair (`bool`, *optional*, defaults to `False`):
                Whether the number of added tokens should be computed in the case of a sequence pair or a single
                sequence.

        Returns:
            `int`: Number of special tokens added to sequences.
        )r  Únum_special_tokens_to_add)r  rU  s     rE   r«  z+TokenizersBackend.num_special_tokens_to_addO  s   € ð& Œ×8Ò8¸Ñ>Ô>Ð>rG   rš  Úskip_special_tokensc                 ó<  — t          |t          ¦  «        r| j                             |¦  «        S g }|rt	          | j        ¦  «        nt	          ¦   «         }|D ]C}t          |¦  «        }||v rŒ|                     | j                             |¦  «        ¦  «         ŒD|S )aì  
        Converts a single index or a sequence of indices in a token or a sequence of tokens, using the vocabulary and
        added tokens.

        Args:
            ids (`int` or `list[int]`):
                The token id (or token ids) to convert to tokens.
            skip_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not to remove special tokens in the decoding.

        Returns:
            `str` or `list[str]`: The decoded token(s).
        )rO   r¤   r  rÌ   rŒ   Úall_special_idsr^   )r  rš  r¬  r+  Úids_to_skipr©   s         rE   Úconvert_ids_to_tokensz'TokenizersBackend.convert_ids_to_tokensd  s¡   € õ �c�3ÑÔð 	4Ø”?×.Ò.¨sÑ3Ô3Ð3Øˆà3FÐQ•c˜$Ô.Ñ/Ô/Ð/ÍCÉEÌEˆØð 	>ð 	>ˆEÝ˜‘J”JˆEØ˜Ð#Ð#ØØ�MŠM˜$œ/×5Ò5°eÑ<Ô<Ñ=Ô=Ð=Ð=ØˆrG   Útextr¨  c                 óH   —  | j         d|||dœ|¤Ž                     ¦   «         S )N)r±  Ú	text_pairr¨  rA   )Ú_encode_plusr+  )r  r±  rU  r¨  r�   s        rE   ÚtokenizezTokenizersBackend.tokenize~  s2   € Ø ˆtÔ Ðl d°dÐOaÐlÐlÐekÐlÐl×sÒsÑuÔuÐurG   Úpadding_strategyrä   rà   rã   rê   rè   c                 ó  ‡— | j         j        Š| j         j        }|t          j        k    r‰�| j                              ¦   «          n<|||j        | j        dœ}‰€d}	nˆfd„|D ¦   «         }	|	|k    r | j         j        di |¤Ž |t          j
        k    r|�| j                              ¦   «          dS dS |t          j        k    r|nd}
|
|�|n| j        | j        | j        | j        |dœ}||k    r | j         j        di |¤Ž dS dS )a•  
        Define the truncation and the padding strategies for fast tokenizers (provided by HuggingFace tokenizers
        library) and restore the tokenizer settings afterwards.

        The provided tokenizer has no padding / truncation strategy before the managed section. If your tokenizer set a
        padding / truncation strategy before, then it will be reset to no padding / truncation when exiting the managed
        section.

        Args:
            padding_strategy ([`~utils.PaddingStrategy`]):
                The kind of padding that will be applied to the input
            truncation_strategy ([`~tokenization_utils_base.TruncationStrategy`]):
                The kind of truncation that will be applied to the input
            max_length (`int`):
                The maximum size of a sequence.
            stride (`int`):
                The stride to use when handling overflow.
            pad_to_multiple_of (`int`, *optional*):
                If set will pad the sequence to a multiple of the provided value. This is especially useful to enable
                the use of Tensor Cores on NVIDIA hardware with compute capability `>= 7.5` (Volta).
            padding_side (`str`, *optional*):
                The side on which the model should have padding applied. Should be selected between ['right', 'left'].
                Default value is picked from the class attribute of the same name.
        N)rà   rã   rå   râ   c                 ó>   •— i | ]}|‰                      |d ¦  «        “ŒS r\   r¨   )rC   rw  r%  s     €rE   rL   z@TokenizersBackend.set_truncation_and_padding.<locals>.<dictcomp>·  s)   ø€ ÐGÐGÐG¸1˜1˜kŸošo¨a°Ñ6Ô6ÐGÐGÐGrG   )ré   râ   Úpad_idrb   rç   rê   rA   )r  r{   rz   r   ÚDO_NOT_TRUNCATEr
  re  rá   r	  r    Ú
DO_NOT_PADÚ
no_paddingÚ
MAX_LENGTHrè   Úpad_token_idrb   ræ   r  )r  r¶  rä   rà   rã   rê   rè   r&  ÚtargetÚcurrentré   r%  s              @rE   Úset_truncation_and_paddingz,TokenizersBackend.set_truncation_and_padding�  si  ø€ ðB ”oÔ0ˆØ”?Ô*ˆàÕ"4Ô"DÒDÐDØÐ&Ø”×-Ò-Ñ/Ô/Ð/øð )Ø Ø/Ô5Ø!Ô1ð	ð ˆFð Ð"Ø��àGÐGÐGÐGÀÐGÑGÔG�à˜&Ò Ð Ø1�”Ô1Ð;Ð;°FÐ;Ð;Ð;à�Ô9Ò9Ð9ØÐ#Ø”×*Ò*Ñ,Ô,Ð,Ð,Ð,ð $Ð#ð $4µÔ7QÒ#QÐ#Q�Z�ZÐW[ˆFà Ø-9Ð-E˜\˜\È4ÔK\ØÔ+Ø!œ^Ø#Ô5Ø&8ðð ˆFð ˜6Ò!Ð!Ø.�”Ô.Ð8Ð8°Ð8Ð8Ð8Ð8Ð8ð "Ð!rG   r   r³  Úis_split_into_wordsÚreturn_tensorsr  c                 ó   ‡ ‡‡‡‡‡‡‡‡#— d„ } ||¦  «        st          d¦  «        ‚|� ||¦  «        st          d¦  «        ‚|rAt          |t          t          f¦  «        o#|o!t          |d         t          t          f¦  «        }nt          |t          t          f¦  «        }|r˜t          |t          ¦  «        rt          d¦  «        ‚|�Pt          |¦  «        t          |¦  «        k    r0t          dt          |¦  «        › dt          |¦  «        › d�¦  «        ‚|�t          t          ||¦  «        ¦  «        n|}n
|r||fgn|g}t          |t          t          f¦  «        s t          dt          |¦  «        › d	�¦  «        ‚‰  	                    |||||	|
¬
¦  «         |€‰ j
        }‰ j        j        |k    r|‰ j        _        ‰ j                             |||¬¦  «        }ˆˆˆˆˆˆˆ ˆfd„|D ¦   «         }i }|d         d         D ]Š#ˆ#fd„|D ¦   «         }||‰#<   Œd„ |D ¦   «         }‰r;g }t          |¦  «        D ]$\  }\  }} ||gt          |d         ¦  «        z  z  }Œ%||d<   |d         D ]}!‰                      |!|‰¦  «         Œt!          |||¬¦  «        }"|s5|€3‰s1t!          d„ |"                     ¦   «         D ¦   «         |"j        ¦  «        }"|"S )Nc                 ór  — t          | t          ¦  «        rdS t          | t          t          f¦  «        �rt	          | ¦  «        dk    rdS t          | d         t          ¦  «        rdS t          | d         t          t          f¦  «        r¬t	          | d         ¦  «        dk    s!t          | d         d         t          ¦  «        rdS t          | d         d         t          t          f¦  «        rFt	          | d         d         ¦  «        dk    p&t          | d         d         d         t          ¦  «        S dS dS dS )NTr   F)rO   rS   rP   rB   r­   )rÿ   s    rE   Ú_is_valid_text_inputz<TokenizersBackend._encode_plus.<locals>._is_valid_text_inputä  s  € Ý˜!�SÑ!Ô!ð Ø�tÝ˜A¥¥e˜}Ñ-Ô-ñ Ý�q‘6”6˜Q’;�;Ø˜4Ý  !¤¥cÑ*Ô*ð 
!Ø˜4Ý  !¤¥t­U mÑ4Ô4ð !Ý˜1˜Qœ4‘y”y A’~�~­°A°a´D¸´G½SÑ)AÔ)A�~Ø#˜tÝ# A a¤D¨¤G­dµE¨]Ñ;Ô;ð %Ý" 1 Q¤4¨¤7™|œ|¨qÒ0ÐOµJ¸qÀ¼tÀA¼wÀq¼zÍ3Ñ4OÔ4OÐOà$˜uà ˜5à�urG   zêtext input must be of type `str` (single example), `list[str]` (batch or single pretokenized example) or `list[list[str]]` (batch of pretokenized examples) or `list[tuple[list[str], list[str]]]` (batch of pretokenized sequence pairs).r   zdwhen tokenizing batches of text, `text_pair` must be a list or tuple with the same length as `text`.zbatch length of `text`: z- does not match batch length of `text_pair`: ú.z:batch_text_or_text_pairs has to be a list or a tuple (got ú))r¶  rä   rà   rã   rê   rè   )r¨  Úis_pretokenizedc                 óJ   •— g | ]}‰                      |‰‰‰‰‰‰‰	¬ ¦  «        ‘Œ S ))r/   r‹  rŒ  r�  rŽ  r�  r�  r‘  )rŸ  )
rC   r/   rŒ  r�  r�  r�  rŽ  r‹  r  r‘  s
     €€€€€€€€rE   rF   z2TokenizersBackend._encode_plus.<locals>.<listcomp>9  sX   ø€ ð  
ð  
ð  
ð ð ×"Ò"Ø!Ø&;Ø&;Ø*CØ+EØ'=Ø+Øð #ñ 	ô 	ð 
ð  
ð  
rG   c                 ó0   •— g | ]\  }}|‰         D ]}|‘ŒŒS rA   rA   )rC   rD   rÝ   r›   r�   s       €rE   rF   z2TokenizersBackend._encode_plus.<locals>.<listcomp>J  s.   ø€ ÐNÐNÐN™7˜4 ÀDÈÄIÐNÐN¸q�QÐNÐNÐNÐNrG   c                 ó"   — g | ]\  }}|D ]}|‘ŒŒS rA   rA   )rC   rÝ   rD   r›   s       rE   rF   z2TokenizersBackend._encode_plus.<locals>.<listcomp>L  s)   € ÐSÐSÐS¡W Q¨ÈdÐSÐSÈ˜qÐSÐSÐSÐSrG   r•  Úoverflow_to_sample_mapping)Útensor_typec                 ó†   — i | ]>\  }}|t          |¦  «        d k    r#t          |d          t          ¦  «        r|d          n|“Œ?S rN   )r­   rO   rP   )rC   r�   re  s      rE   rL   z2TokenizersBackend._encode_plus.<locals>.<dictcomp>]  sW   € ð ð ð á"˜˜Uð ¥c¨%¡j¤j°1¢n n½ÀEÈ!ÄHÍdÑ9SÔ9S n˜% œ(˜(ÐY^ðð ð rG   )r  rO   rP   rB   rS   Ú	TypeErrorr­   Úzipr1   rÁ  r  r  r  Úencode_batchr€   Ú&_eventual_warn_about_too_long_sequencer   r«   r�  )$r  r±  r³  r¨  r¶  rä   rà   rã   rÂ  rê   rè   rÃ  r‹  rŒ  r�  rŽ  r�  r�  r‘  r  r�   rÆ  Ú
is_batchedÚbatch_text_or_text_pairsr�  Útokens_and_encodingsÚsanitized_tokensÚstackÚsanitized_encodingsrÍ  rJ   ÚtoksrÝ   r•  Úbatched_outputr�   s$   `           ```````                @rE   r´  zTokenizersBackend._encode_plusÌ  s  øøøøøøøøø€ ð0	ð 	ð 	ð( $Ð# DÑ)Ô)ð 	ÝðWñô ð ð
 Ð Ð)=Ð)=¸iÑ)HÔ)HÐ ÝðWñô ð ð ð 	9Ý# D­4µ¨-Ñ8Ô8Ðh¸TÐhÅjÐQUÐVWÔQXÕ[_ÕafÐZgÑFhÔFhˆJˆJå# D­4µ¨-Ñ8Ô8ˆJàð 	Tå˜)¥SÑ)Ô)ð Ýðñô ð ð Ð$­¨T©¬µc¸)±n´nÒ)DÐ)DÝ ð*­s°4©y¬yð *ð *Ý˜I™œð*ð *ð *ñô ð ð FOÐEZ¥t­C°°iÑ,@Ô,@Ñ'AÔ'AÐ'AÐ`dÐ$Ð$ð ?HÐ'S¨¨yÐ(9Ð':Ð':ÈdÈVÐ$õ Ð2µU½D°MÑBÔBð 	ÝØnÍTÐRjÑMkÔMkÐnÐnÐnñô ð ð 	×'Ò'Ø-Ø 3Ø!ØØ1Ø%ð 	(ñ 	
ô 	
ð 	
ð  Ð'Ø#'Ô#<Ð àŒ?Ô0Ð4HÒHÐHØ4HˆDŒOÔ1ð ”O×0Ò0Ø$Ø1Ø/ð 1ñ 
ô 
ˆ	ð 
ð  
ð  
ð  
ð  
ð  
ð  
ð  
ð  
ð  
ð  
ð &ð 
ñ  
ô  
Ðð ÐØ'¨Ô*¨1Ô-ð 	*ð 	*ˆCØNÐNÐNÐNÐ&:ÐNÑNÔNˆEØ$)Ð˜SÑ!Ð!ØSÐSÐ0DÐSÑSÔSÐð %ð 	XØ)+Ð&Ý )Ð*>Ñ ?Ô ?ð Kð K‘�‘9�D˜!Ø*¨q¨cµC¸¸[Ô8IÑ4JÔ4JÑ.JÑJÐ*Ð*Ø=WÐÐ9Ñ:à)¨+Ô6ð 	Xð 	XˆIØ×7Ò7¸	À:ÈwÑWÔWÐWÐWå&Ð'7Ð9LÐZhÐiÑiÔiˆð ð 	˜nÐ4Ð=VÐ4Ý*ðð à&4×&:Ò&:Ñ&<Ô&<ðñ ô ð Ô(ñô ˆNð ÐrG   r+  c                 ó‚   — | j         j        �| j         j                             |¦  «        nd                     |¦  «        S )NrR   )rˆ  rŠ  Údecoder;  )r  r+  s     rE   Úconvert_tokens_to_stringz*TokenizersBackend.convert_tokens_to_stringf  s@   € ð Ô%Ô-Ð9ð Ô"Ô*×1Ò1°&Ñ9Ô9Ð9à—’˜&Ñ!Ô!ð	
rG   Ú	token_idsÚclean_up_tokenization_spacesc                 ó¼  — |                      dd ¦  «         t          |t          ¦  «        r|g}t          |t          ¦  «        r|d         }| j                             ||¬¦  «        }|�|n| j        }|rgt          | j        j	        ¦  «        j
        dk    r0| j        s)t                               d| j        j
        › d�¦  «         n|                      |¦  «        }|S )NÚuse_source_tokenizerr•  )r¬  r   z=Ignoring clean_up_tokenization_spaces=True for BPE tokenizer aE  . The clean_up_tokenization post-processing step is designed for WordPiece tokenizers and is destructive for BPE (it strips spaces before punctuation). Set clean_up_tokenization_spaces=False to suppress this warning, or set clean_up_tokenization_spaces_for_bpe_even_though_it_will_corrupt_output=True to force cleanup anyway.)rm   rO   r¤   rl   r  rÝ  rà  r1   rˆ  r0   r   ÚGclean_up_tokenization_spaces_for_bpe_even_though_it_will_corrupt_outputr‰   Úwarning_oncer.  Úclean_up_tokenization)r  rß  r¬  rà  r�   r±  s         rE   Ú_decodezTokenizersBackend._decodem  s  € ð 	�
Š
Ð)¨4Ñ0Ô0Ð0å�i¥Ñ%Ô%ð 	$Ø"˜ˆIÝ�i¥Ñ&Ô&ð 	/Ø! +Ô.ˆIØŒ×%Ò% iÐEXÐ%ÑYÔYˆð ,Ð7ð )Ð(àÔ2ð 	%ð
 (ð 	8õ
 �TÔ+Ô1Ñ2Ô2Ô;¸uÒDÐDØÔdð Eõ ×#Ò#ð-ØœÔ/ð-ð -ð -ñô ð ð ð ×1Ò1°$Ñ7Ô7�àˆrG   Ú
file_namesc                 óÂ   — t          |¦  «        }t          j                             ||r|dz   ndt          z   ¦  «        }| j                             |¦  «         ||fz   }|S )Nr8  r×   )rS   rn   ro   r;  ÚTOKENIZER_FILErˆ  Úsave)r  r5  rç  r@  r6  r&   s         rE   Ú_save_pretrainedz"TokenizersBackend._save_pretrained˜  sl   € õ ˜^Ñ,Ô,ˆåœŸšØ°oÐM˜_¨sÑ2Ð2È2ÕQ_Ñ_ñ
ô 
ˆð 	Ô×#Ò# NÑ3Ô3Ð3Ø >Ð"3Ñ3ˆ
àÐrG   c           	      ó  ‡‡— t          j        | j                             ¦   «         ¦  «        }|                     d¦  «        }|                     d¦  «        }	d}
|d         d         dk    ri |d         d<   g |d         d<   n«|d         d         d	k    r^|d         d
         �O|d         d
         }|d         d         |         d         }
‰�|
‰v r‰|
         }
d|d         d
<   |
dgg|d         d<   n;|d         d         dv ri |d         d<   nt          d|d         d         › d�¦  «        ‚‰�7d|d         v r-|d         d         ‰v r‰|d         d                  |d         d<   t          j        t          j        |¦  «        ¦  «        Šg }|D ]}|                     dd¦  «        }|                     dd¦  «        }|d         d         d	k    r|sŒC‰�|d         ‰v r‰|d                  |d<   | 	                    t          d*i |¤Ž¦  «         Œ€|�|                     |¦  «         |d         d         dk    r#d|vr|d         d         �|d         d         |d<   |d         d         dk    r#d|vr|d         d         �|d         d         |d<   |d         d         d	k    r|
�|
|d<   |d         �t|d         d         dk    sA|d         d         dk    rPd|d         v rFt          d„ |d         d         D ¦   «         ¦  «        r!t          j                             ¦   «         |d<   t           |d         d                  } |d*||dœ|¤Ž}‰                     |||¬¦  «         |	��,t          j        ‰                     ¦   «         ¦  «        }d|	v rˆ|	d         D ]}|	d         |         d         }‰�ˆfd „|D ¦   «         }||	d         |         d<   |D ](}‰                     |¦  «        }|€t          d!¦  «        ‚Œ)ˆfd"„|D ¦   «         |	d         |         d#<   Œ€d$D ]L}||	v rF|	|         \  }}‰�|‰v r‰|         }‰                     |¦  «        }|€t          d!¦  «        ‚||g|	|<   ŒM|	|d<   t          j        t          j        |¦  «        ¦  «        Š| j                             ¦   «         }t*          j        D ]’}t/          | |¦  «        �€t/          | |¦  «        }‰�|‰v r‰|         }| j                             |d¦  «        }t5          |t          ¦  «        r-t          ||j        |j        |j        |j        d%¬&¦  «        ||<   Œ�|||<   Œ“| j        r| j                             ¦   «         ng }|�|                     |¦  «         tA          |¦  «        dk    r||d'<   ‰|d(<   	  | j!        d*i |¤ŽS # tD          $ rH}d)tG          |¦  «        v r1|                     d(d¦  «          | j!        d*i |¤Ž}‰|_        |cY d}~S ‚ d}~ww xY w)+uf  
        Trains a tokenizer on a new corpus with the same defaults (in terms of special tokens or tokenization pipeline)
        as the current one.

        Args:
            text_iterator (generator of `list[str]`):
                The training corpus. Should be a generator of batches of texts, for instance a list of lists of texts
                if you have everything in memory.
            vocab_size (`int`):
                The size of the vocabulary you want for your tokenizer.
            length (`int`, *optional*):
                The total number of sequences in the iterator. This is used to provide meaningful progress tracking
            new_special_tokens (list of `str` or `AddedToken`, *optional*):
                A list of new special tokens to add to the tokenizer you are training.
            special_tokens_map (`dict[str, str]`, *optional*):
                If you want to rename some of the special tokens this tokenizer uses, pass along a mapping old special
                token name to new special token name in this argument.
            kwargs (`dict[str, Any]`, *optional*):
                Additional keyword arguments passed along to the trainer from the ðŸ¤— Tokenizers library.

        Returns:
            [`PreTrainedTokenizerFast`]: A new tokenizer of the same type as the original one, trained on
            `text_iterator`.

        r4   r5   Nr0   r1   r   r2   r3   r   rÈ   r   g        )r$   r%   z;This method does not support this type of tokenizer (found z-) only BPE, Unigram, WordLevel and WordPiece.rc   r   Úidr£   Úcontinuing_subword_prefixÚend_of_word_suffixr  Ú	ByteLevelr;   Úpretokenizersc              3   ó.   K  — | ]}|d          dk    V — ŒdS )r1   rð  NrA   )rC   Úpretokenizers     rE   ú	<genexpr>z<TokenizersBackend.train_new_from_iterator.<locals>.<genexpr>  s@   è è € ð ð à$ð ! Ô(¨KÒ7ðð ð ð ð ð rG   Úinitial_alphabet)r-  rV  )ré   ÚtrainerrV  r+  c                 ó<   •— g | ]}‰                      ||¦  «        ‘ŒS rA   r¨   )rC   rK   Úspecial_tokens_maps     €rE   rF   z=TokenizersBackend.train_new_from_iterator.<locals>.<listcomp>#  s*   ø€ Ð![Ð![Ð![È5Ð"4×"8Ò"8¸ÀÑ"FÔ"FÐ![Ð![Ð![rG   zQAttempted to set a token in the post processor that does not exist in the mappingc                 ó:   •— g | ]}‰                      |¦  «        ‘ŒS rA   )r¡  )rC   rK   rÙ   s     €rE   rF   z=TokenizersBackend.train_new_from_iterator.<locals>.<listcomp>,  s)   ø€ ÐCuÐCuÐCuÐejÀI×DYÒDYÐZ_ÑD`ÔD`ÐCuÐCuÐCurG   rš  )rŽ   rN  T)Úsingle_wordÚlstripÚrstripri  r   rj   r,   z7multiple values for keyword argument 'tokenizer_object'rA   )$ru   Úloadsr  Úto_strrm   r  rr   rx   ry   r^   r   r]   ÚanyÚpre_tokenizers_fastrð  ÚalphabetÚMODEL_TO_TRAINER_MAPPINGÚtrain_from_iteratorr¡  r  r  r   ÚSPECIAL_TOKENS_ATTRIBUTESr�   r  rw   rO   rú  rû  rü  ri  rj   r­   r.  rÐ  rS   )r  Útext_iteratorr-  ré   Únew_special_tokensrø  r�   r”   r4   r5   rc   rÈ   rV  Úadded_tokenr   rÝ   Útrainer_classrö  Útrained_tokenizer_jsonr�   r+  rK   r¥   rµ   Úspecial_token_fullrj   r›   Únew_tokenizerrÙ   s        `                      @rE   Útrain_new_from_iteratorz)TokenizersBackend.train_new_from_iterator©  sÀ  øø€ õD œ D¤O×$:Ò$:Ñ$<Ô$<Ñ=Ô=ˆà%×)Ò)¨.Ñ9Ô9ˆà'×+Ò+Ð,<Ñ=Ô=ˆàˆ	à˜'Ô" 6Ô*¨eÒ3Ð3Ø/1ˆN˜7Ô# GÑ,Ø02ˆN˜7Ô# HÑ-Ð-Ø˜GÔ$ VÔ,°	Ò9Ð9Ø˜gÔ& xÔ0Ð<Ø'¨Ô0°Ô:�Ø*¨7Ô3°GÔ<¸VÔDÀQÔG�	Ø%Ð1°iÐCUÐ6UÐ6UØ 2°9Ô =�IØ45�˜wÔ'¨Ñ1Ø5>ÀÐ4DÐ3E�˜wÔ'¨Ñ0øØ˜GÔ$ VÔ,Ð0JÐJÐJØ/1ˆN˜7Ô# GÑ,Ð,åð>ÈnÐ]dÔNeÐflÔNmð >ð >ð >ñô ð ð Ð*Ø˜~¨gÔ6Ð6Ð6Ø˜wÔ'¨Ô4Ð8JÐJÐJà3EÀnÐU\ÔF]Ð^iÔFjÔ3kˆN˜7Ô# KÑ0å!Ô*­4¬:°nÑ+EÔ+EÑFÔFˆ	ð ˆØ'ð 	=ð 	=ˆKØ!—o’o i°Ñ6Ô6ˆGØ—’  dÑ+Ô+ˆAØ˜gÔ& vÔ.°)Ò;Ð;ÀGÐ;ØØ!Ð-°+¸iÔ2HÐL^Ð2^Ð2^Ø);¸KÈ	Ô<RÔ)S�˜IÑ&Ø×!Ò!¥*Ð";Ð";¨{Ð";Ð";Ñ<Ô<Ð<Ð<àÐ)Ø×!Ò!Ð"4Ñ5Ô5Ð5ð ˜7Ô# FÔ+¨uÒ4Ð4Ø+°6Ð9Ð9Ø˜wÔ'Ð(CÔDÐPà2@ÀÔ2IÐJeÔ2fˆFÐ.Ñ/à˜7Ô# FÔ+¨uÒ4Ð4Ø$¨FÐ2Ð2Ø˜wÔ'Ð(<Ô=ÐIà+9¸'Ô+BÐCWÔ+XˆFÐ'Ñ(Ø˜'Ô" 6Ô*¨iÒ7Ð7¸IÐ<QØ"+ˆF�;ÑØ˜/Ô*Ð6à˜Ô/°Ô7¸;ÒFÐFØ! /Ô2°6Ô:¸jÒHÐHØ# ~°oÔ'FÐFÐFÝð ð à(6°Ô(GÈÔ(Xðñ ô ñ ô ð Gõ .AÔ-J×-SÒ-SÑ-UÔ-U�Ð)Ñ*å0°ÀÔ1HÈÔ1PÔQˆØ�-Ð_¨:ÀnÐ_Ð_ÐX^Ð_Ð_ˆØ×%Ò% m¸FÈGÐ%ÑTÔTÐTàÑ%Ý%)¤Z°	×0@Ò0@Ñ0BÔ0BÑ%CÔ%CÐ"à >Ð1Ð1Ø)Ð*:Ô;ð vð v�CØ+Ð,<Ô=¸cÔBÀ8ÔL�FØ)Ð5Ø![Ð![Ð![Ð![ÐTZÐ![Ñ![Ô![˜ØFL�NÐ#3Ô4°SÔ9¸(ÑCØ!'ð ð ˜Ø#,×#8Ò#8¸Ñ#?Ô#?˜Ø#Ð+Ý",Ø sñ#ô #ð ð ,ð
 DvÐCuÐCuÐCuÐntÐCuÑCuÔCu�NÐ#3Ô4°SÔ9¸%Ñ@Ð@à!/ð 
Fð 
F�Ø  NÐ2Ð2Ø-¨mÔ<‘H�E˜1Ø)Ð5¸%ÐCUÐ:UÐ:UØ 2°5Ô 9˜Ø(×4Ò4°UÑ;Ô;�HØÐ'Ý(Øoñô ð ð 6;¸HÐ4E�N =Ñ1øà7EÐ"Ð#3Ñ4Ý%Ô.­t¬zÐ:PÑ/QÔ/QÑRÔRˆIàÔ!×&Ò&Ñ(Ô(ˆå,ÔFð 	2ð 	2ˆEÝ�t˜UÑ#Ô#Ð/Ý '¨¨eÑ 4Ô 4�Ø%Ð1°mÐGYÐ6YÐ6YØ$6°}Ô$E�Mà%)Ô%=×%AÒ%AÀ%ÈÑ%NÔ%NÐ"ÝÐ0µ*Ñ=Ô=ð 2å$.Ø%Ø$6Ô$BØ1Ô8Ø1Ô8Ø#5Ô#@Ø $ð%ñ %ô %�F˜5‘M�Mð %2�F˜5‘Møð DHÔC\Ðd˜tÔ8×=Ò=Ñ?Ô?Ð?ÐbdÐØÐ)Ø ×'Ò'Ð(:Ñ;Ô;Ð;ÝÐ#Ñ$Ô$ qÒ(Ð(Ø-AˆFÐ)Ñ*ð &/ˆÐ!Ñ"ð	Ø!�4”>Ð+Ð+ FÐ+Ð+Ð+øÝð 	ð 	ð 	àHÍCÐPQÉFÌFÐRÐRð —
’
Ð-¨tÑ4Ô4Ð4Ø . ¤Ð 8Ð 8°Ð 8Ð 8�Ø+4�Ô(Ø$Ð$Ð$Ð$Ð$Ð$Ð$ð øøøøð	øøøs$   Ö!V. Ö.
X Ö8<W;×4X ×:W;×;X c
           
      ó@  ‡‡— ddl Šddlm} ddlm} ddlm}mŠ  |d¬¦  «        dt          d	t          fˆˆfd
„¦   «         }|st          ¦   «         rd}|��G|s|�sB ||¦  «        �r6 ||d|||dd|¬¦  «        }d}|�Òt          |d¬¦  «        5 }t          j        |¦  «        }ddd¦  «         n# 1 swxY w Y   |                     d¦  «        }|                     d¦  «        }|r7|                     |¦  «        |                     d¦  «        k     r|r|�|dvr|S n0|r.|                     |¦  «        |                     d¦  «        k    r|S d}|s|�sH ||¦  «        �r<|rd|v rt!          |d|d         ¦  «         |	€At#          |dd¦  «        s0t!          |dd¦  «         t$                               d|› d�¦  «         nÜ|	du st#          |dd¦  «        rÇt!          |dd¦  «         ddl}|j                             |                     d¦  «        d¬¦  «        }|j        }t3          ||j        j        ¦  «        r||j        d<   nWt3          ||j        j        ¦  «        r|j                             dd¬¦  «        }|j                             ||g¦  «        |_        |S )af  
        Patches mistral related tokenizers with incorrect regex if detected
            1) Local file with an associated config saved next to it
                >> Model type one of the mistral models (on older versions)
            2) Remote models on the hub from official mistral models
                >> Tags including `base_model:.*mistralai`
        r   N)Ú	lru_cache)Úversion)r   r"   é€   )ÚmaxsizeÚmodel_idrZ   c                 óÖ   •— 	  ‰¦   «                               | ¦  «        }n# t          $ r Y dS w xY w|j        �0‰                     dd                     |j        ¦  «        ¦  «        rdS dS )NFzbase_model:.*mistralair×   T)Ú
model_inforˆ   ÚtagsÚsearchr;  )r  r0   r"   Úres     €€rE   Úis_base_mistralz?TokenizersBackend._patch_mistral_regex.<locals>.is_base_mistralŠ  s~   ø€ ðØ˜™œ×+Ò+¨HÑ5Ô5��øÝð ð ð à�u�uðøøøð ŒzÐ%Ø—9’9Ð5°r·w²w¸u¼zÑ7JÔ7JÑKÔKð  Ø˜4Ø�5s   ƒ! ¡
/®/Tzconfig.jsonF)Ú	cache_dirrK   Úlocal_files_onlyÚ%_raise_exceptions_for_missing_entriesÚ'_raise_exceptions_for_connection_errorsÚ_commit_hashr-   r.   Útransformers_versionr•   z5.0.0)rE  Úmistral3ÚvoxtralÚ	ministralÚpixtralr  z$The tokenizer you are loading from 'a  ' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue.zÓ[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n/]*|\s*[\r\n]+|\s+(?!\S)|\s+Úisolated)ÚpatternÚbehavior)rÒ   Ú	use_regex)r  Ú	functoolsr  Ú	packagingr  Útransformers.utils.hubr   r"   rS   Úboolr   rt   ru   rv   rw   ÚparseÚsetattrr�   r‰   rŠ   rì   Úpre_tokenizersÚSplitÚRegexr  rO   r;   Ú	Metaspacerð  )rŽ   rÙ   Úpretrained_model_name_or_pathrK   r  r  r  Úis_localr  r  r�   r  r  r   r  Ú_config_fileÚmistral_config_detectedÚfÚ_configr  Útransformers_model_typerì   Úsplit_pretokenizerÚcurrent_pretokenizerr"   r  s                           @@rE   r  z&TokenizersBackend._patch_mistral_regexn  s  øø€ ð* 	ˆ	ˆ	ˆ	Ø'Ð'Ð'Ð'Ð'Ð'à%Ð%Ð%Ð%Ð%Ð%à>Ð>Ð>Ð>Ð>Ð>Ð>Ð>à	ˆ˜3Ð	Ñ	Ô	ð		¥cð 		­dð 		ð 		ð 		ð 		ð 		ð 		ñ 
 Ô	ð		ð ð 	�Ñ0Ô0ð 	ØˆHà(Ñ4Øð 5Ø%ñ 5Ø*9¨/Ð:WÑ*XÔ*Xñ 5ð '˜;Ø-ØØ#ØØ!1Ø6;Ø8=Ø)ð	ñ 	ô 	ˆLð ',Ð#ØÐ'Ý˜,°Ð9Ñ9Ô9ð +¸QÝ"œi¨™lœl�Gð+ð +ð +ñ +ô +ð +ð +ð +ð +ð +ð +øøøð +ð +ð +ð +à'.§{¢{Ð3IÑ'JÔ'JÐ$Ø*1¯+ª+°lÑ*CÔ*CÐ'ð
 (ð %¨G¯MªMÐ:NÑ,OÔ,OÐRY×R_ÒR_Ð`gÑRhÔRhÒ,hÐ,hà ð)à3Ð?Ø3ð ðð ð  )Ð(øØ)ð %¨g¯mªmÐ<PÑ.QÔ.QÐU\×UbÒUbÐcjÑUkÔUkÒ.kÐ.kØ$Ð$à*.Ð'à&ð *¨xñ *¸O¸OÐLiÑ<jÔ<jñ *àð ^Ð#6¸+Ð#EÐ#EÝ˜IÐ':¸KÐH[Ô<\Ñ]Ô]Ð]ð %Ð,µW¸YÐH[Ð]bÑ5cÔ5cÐ,Ý˜IÐ':¸EÑBÔBÐBÝ—N’NðeÐ?\ð eð eð eñô ð ð ð
 '¨$Ð.Ð.µ'¸)ÐEXÐZ_Ñ2`Ô2`Ð.Ý˜IÐ':¸DÑAÔAÐAØ%Ð%Ð%Ð%à)3Ô)B×)HÒ)HØ *× 0Ò 0ð sñ!ô !ð ",ð	 *Iñ *ô *Ð&ð ,5Ô+BÐ(å!Ð"6¸
Ô8QÔ8ZÑ[Ô[ð à5G˜	Ô/°Ñ2Ð2õ &Ð&:¸JÔ<UÔ<_Ñ`Ô`ð Ø3=Ô3L×3VÒ3VØ16À%ð 4Wñ 4ô 4Ð0ð
 3=Ô2K×2TÒ2Tà 2Ø 4ðñ3ô 3˜	Ô/ð Ðs   ÂB1Â1B5Â8B5)Fr\   )NNFN)NNFFFFT)NF)FN)NN)NNN)NNFNFNN)Fr   Ú
__module__Ú__qualname__Ú__doc__r<  r3  r0   r  Úclassmethodrž   rS   rl   r   r‹   r‡   r+   Úpropertyr*  r1  r4  rB   r?  rn   ÚPathLikerQ  r  rî   rí   Úsetterrk  r¤   r-  rq  r2   r  r   r¦   Ú_added_tokens_encoderÚ_added_tokens_decoderr‚  r„  r†  rr   rˆ  ÚDecoderFastrŠ  ÚEncodingFastrP   rŸ  r£  r¥  r©  r«  r°  rµ  r    r   rÁ  r»  rº  r   r   r   r´  rÞ  ræ  rë  r  r  Ú__classcell__)r.  s   @rE   r)   r)   W   sz	  ø€ € € € € ð
ð 
ð *ÐØ€EØ€Jàðdð dð dñ „[ðdðL ð!°ð !À4ÈÈSÈÄ>ð !ÐVZÐ[^Ð`cÐ[cÔVdð !ð !ð !ñ „[ð!ðF ð9°Sð 9ÈÈSÐRUÈXÌð 9Ð[_Ð`cÐehÐ`hÔ[ið 9ð 9ð 9ñ „[ð9ðvh)ð h)ð h)ð h)ð h)ðT ð˜ð ð ð ñ „Xðð ð¨ð ð ð ñ „Xðð!ð !¨cð !ÀCÈ$ÁJð !ÐZ_Ð`cÔZdð !ð !ð !ð !ð  &*Ø&*Ø!Ø"&ðB
ð B
à˜bœkÑ)ðB
ð ˜d‘{ðB
ð ˜t™ð	B
ð
 ðB
ð ˜4‘ZðB
ð 
ˆs�CˆxŒðB
ð B
ð B
ð B
ð B
ð B
ðH
ð 
ð 
ð4 ð6ð 6ñ „Xð6ð ð6ð 6ñ „Xð6ð Ôð%ð %ñ Ôð%ð Ôð%ð %ñ Ôð%ð)ð )ð )ð@ ðG˜Cð Gð Gð Gñ „XðGðA˜4  S œ>ð Að Að Að Að ð �t˜C ˜H”~ð  ð  ð  ñ „Xð ð ðn d¨3°¨8¤nð nð nð nñ „Xðnð ð: d¨3°
¨?Ô&;ð :ð :ð :ñ „Xð:ð 1ÐØ0Ððn  c¨3 h¤ð nð nð nð nð˜$ð ð ð ð ðF˜ð Fð Fð Fð Fð ð =ð ð ð ñ „Xðð ð'˜ð 'ð 'ð 'ñ „Xð'ð .2Ø-1Ø*/Ø+0Ø',Ø#Øð-(ð -(àð-(ð  $ d™{ð-(ð  $ d™{ð	-(ð
 $(ð-(ð %)ð-(ð !%ð-(ð ð-(ð ð-(ð 
ˆt�C˜�HŒ~˜t LÔ1Ð1Ô	2ð-(ð -(ð -(ð -(ð^¸ð Àð ð ð ð ð7¨#ð 7°#¸±*ð 7ð 7ð 7ð 7ð6ð 6 d¨3°Ñ+;Ô&<ð 6ÐWZð 6ð 6ð 6ð 6ð?ð ?¨dð ?¸sð ?ð ?ð ?ð ?ð*ð ¨¨t°C¬y©ð Ètð Ð`cÐfjÐknÔfoÑ`oð ð ð ð ð4vð v˜Sð v¨¨d©
ð vÈtð vÐjnÐorÔjsð vð vð vð vðI9à)ðI9ð 0ðI9ð ð	I9ð
 ðI9ð   $™JðI9ð ˜D‘jðI9ð I9ð I9ð I9ð\ gkØ#'Ø,;Ô,FØ2DÔ2TØ!%ØØ$)Ø)-Ø#'Ø&*Ø-1Ø-1Ø*/Ø+0Ø',Ø#ØØ,0ð)Xð XàÐ+Ñ+¨d°9¬oÑ=ÀÐEVÔ@WÑWðXð Ð0Ñ0°4¸	´?ÑBÀTÐJ[ÔE\Ñ\Ð_cÑcðXð !ð	Xð
 *ðXð 0ðXð ˜$‘JðXð ðXð "ðXð   $™JðXð ˜D‘jðXð ˜t™ðXð  $ d™{ðXð  $ d™{ðXð $(ðXð  %)ð!Xð" !%ð#Xð$ ð%Xð& ð'Xð( # T™kð)Xð, 
ð-Xð Xð Xð Xðt
¨t°C¬yð 
¸Sð 
ð 
ð 
ð 
ð %*Ø48ð	)ð )à˜˜cœ‘?ð)ð "ð)ð '+¨T¡kð	)ð 
ð)ð )ð )ð )ð^ &*Ø&*ðð à˜bœkÑ)ðð ˜#˜s˜(”Oðð ˜d‘{ð	ð
 ˜t™ðð 
ˆs�CˆxŒðð ð ð ð* ØØðCð Cð Cð CðJ ð
 ØØØØØØðBð Bð Bñ „[ðBð Bð Bð Bð BrG   r)   )Gr<  r  ru   rn   Úcollectionsr   Úcollections.abcr   Úshutilr   Útypingr   Útokenizers.pre_tokenizersr-  r   Úhuggingface_hubr   rì   r   r	   r
   rD  r   rr   Útokenizers.decodersr   rC  Útokenizers.modelsr   r   Útokenizers.trainersr   r   r   r   r)  r   rª   r   Úintegrations.ggmlr   Úintegrations.mistral.constantsr   Úmodeling_gguf_pytorch_utilsr   Útokenization_utils_baser   r   r   r   r   r   r   Úutilsr    r!   r"   r#   Ú
get_loggerr   r‰   ré  ÚSPECIAL_TOKENS_MAP_FILEÚTOKENIZER_CONFIG_FILEÚTIKTOKEN_VOCAB_FILEÚADDED_TOKENS_FILEr†   r  r<  r)   ÚPreTrainedTokenizerFastrA   rG   rE   ú<module>rZ     sÑ  ððð ð
 €€€Ø €€€Ø 	€	€	€	Ø #Ð #Ð #Ð #Ð #Ð #Ø $Ð $Ð $Ð $Ð $Ð $Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø +Ð +Ð +Ð +Ð +Ð +Ø -Ð -Ð -Ð -Ð -Ð -Ð -Ð -Ø /Ð /Ð /Ð /Ð /Ð /Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø *Ð *Ð *Ð *Ð *Ð *Ð *Ð *Ø ^Ð ^Ð ^Ð ^Ð ^Ð ^Ð ^Ð ^Ð ^Ð ^Ð ^Ð ^à .Ð .Ð .Ð .Ð .Ð .à 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø DÐ DÐ DÐ DÐ DÐ DØ =Ð =Ð =Ð =Ð =Ð =ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð HÐ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GÐ Gð 
ˆÔ	˜HÑ	%Ô	%€ð "€Ø3Ð Ø/Ð Ø'Ð ð (Ð ð (Ð à ð ñ Ð ð ØØ!Ø!ð	ð Ð ð (6ÐEXÐYÐYÐ ð ÐÐ,Ñ-Ô-ðYð Yð Yð Yð YÐ/ñ Yô Yñ .Ô-ðYðz, ,Ð Ð Ð rG   