§
    9²¸j\  ã            
       ór  — d Z ddlZddlZddlZddlmZ ddlmZ ddlm	Z	 ddl
mZmZmZmZmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZmZ ddlmZmZ ddlmZ ddl m!Z!m"Z"  e¦   «         rddl#m$Z$  ej%        e&¦  «        Z'de(ej)        z  de(dz  de(dz  fd„Z*de(ej)        z  de(de(dz  fd„Z+	 d,de(ej)        z  de,dz  de-e,e(dz  f         fd„Z.dddddœZ/de0de-e1e(         e2f         fd „Z3 G d!„ d"¦  «        Z4d#ede(fd$„Z5d#ed%e(ddfd&„Z6d#ed'e(ej)        z  de-e(d(f         fd)„Z7	 d,d*e(de(dz  defd+„Z8dS )-zOConversion between Mistral tekken tokenizers and HuggingFace tokenizer formats.é    N)Ú	lru_cache)ÚPath)Úcopyfile)Ú
AddedTokenÚRegexÚ	TokenizerÚdecodersÚpre_tokenizersÚ
processors)ÚBPEé   )Úbytes_to_unicode)ÚTokenizersBackend)Úcached_fileÚloggingÚrequires_backends)ÚCHAT_TEMPLATE_FILEÚ#LEGACY_PROCESSOR_CHAT_TEMPLATE_FILE)Úis_mistral_common_availableé   )ÚTEKKEN_VOCAB_FILEÚis_tekken_vocab_filename)Ú
TekkenizerÚtekken_fileÚchat_templateÚreturnc                 ó  — |�|S t          | ¦  «        j        }|t          z  }|                     ¦   «         r|                     d¬¦  «        S |t
          z  }|                     ¦   «         rEt          |d¬¦  «        5 }t          j        |¦  «        }ddd¦  «         n# 1 swxY w Y   |d         S t          ¦   «         rK	 ddl
m}  || ¦  «        S # t          $ r,}t                               d| › d|› d	�¦  «         Y d}~dS d}~ww xY wdS )
uˆ  Resolve the chat template to attach during tekken to HF conversion.

    Applies a fixed precedence order:

    1. `chat_template` argument (if not `None`) â€” returned unchanged.
    2. Sibling `chat_template.jinja` file in the same directory as *tekken_file*.
    3. Sibling `chat_template.json` file â€” value of its `"chat_template"` key.
    4. Automatic generation via `mistral_common.integrations.chat_templates` (lazy import).
    5. `None` if none of the above succeed.

    Args:
        tekken_file (`str` or `os.PathLike`): Path to the `tekken.json` file.
        chat_template (`str` or `None`): Explicit chat template string. Only `None`
            triggers the lookup cascade; an empty string is returned as-is.

    Returns:
        Resolved chat template string, or `None`.

    Raises:
        KeyError: If `chat_template.json` exists but does not contain a `"chat_template"` key.
    Núutf-8©Úencodingr   r   )Ú"convert_tokenizer_to_chat_templatez'Failed to generate chat template from 'z': z#. Falling back to no chat template.)r   Úparentr   Úis_fileÚ	read_textr   ÚopenÚjsonÚloadr   Ú9mistral_common.integrations.chat_templates.chat_templatesr!   Ú	ExceptionÚloggerÚwarning_once)	r   r   r"   Ú
jinja_pathÚ	json_pathÚfÚdatar!   Úexcs	            ú{/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/integrations/mistral/tokenizer.pyÚ_resolve_chat_templater2   *   sŸ  € ð. Ð ØÐå�+ÑÔÔ%€Fð Õ,Ñ,€JØ×ÒÑÔð 6Ø×#Ò#¨WÐ#Ñ5Ô5Ð5ð Õ<Ñ<€IØ×ÒÑÔð %Ý�) gÐ.Ñ.Ô.ð 	 °!Ý”9˜Q‘<”<ˆDð	 ð 	 ð 	 ñ 	 ô 	 ð 	 ð 	 ð 	 ð 	 ð 	 ð 	 øøøð 	 ð 	 ð 	 ð 	 à�OÔ$Ð$õ #Ñ$Ô$ð ð
	ðð ð ð ð ð ð 6Ð5°kÑBÔBÐBøÝð 	ð 	ð 	Ý×ÒØr¸+ÐrÐrÈ#ÐrÐrÐrñô ð ð �4�4�4�4�4øøøøð		øøøð ˆ4s*   Á<BÂB!Â$B!Â?C Ã
DÃ!DÄDÚpretrained_model_name_or_pathÚfilenamec                 ó$   — t          | |fdddœ|¤ŽS )aB  Return the resolved path for *filename* inside *pretrained_model_name_or_path*, or `None`.

    Exceptions for missing entries and connection errors are suppressed so that
    callers can treat a `None` return as "file not found / not reachable".

    Args:
        pretrained_model_name_or_path (`str` or `os.PathLike`): Model id or local directory.
        filename (`str`): File name to look up inside the checkpoint.
        **cache_kwargs: Forwarded to [`~utils.cached_file`].

    Returns:
        Resolved local path string, or `None` if the file could not be found.
    F)Ú%_raise_exceptions_for_missing_entriesÚ'_raise_exceptions_for_connection_errors)r   )r3   r4   Úcache_kwargss      r1   Ú_probe_filer9   d   s8   € õ$ Ø%Øðð /4Ø05ð	ð ð
 ðð ð ó    Úmistral_formatc                 ób  — |du rdS |                      dd¦  «         |                      dd¦  «         |du rQt          ¦   «         st          d¦  «        ‚t          | t          fi |¤Ž}|€t          dt          › d	| › d
�¦  «        ‚d|fS t          ¦   «         sdS t          | t          fi |¤Ž}|du|fS )uÔ  Resolve whether to use `MistralCommonBackend` for tokenization.

    Probes for `tekken.json` in the checkpoint directory and checks whether
    `mistral-common` is installed to determine the appropriate tokenizer backend.

    Args:
        pretrained_model_name_or_path (`str` or `os.PathLike`):
            This can be either:

            - a string, the *model id* of a pretrained model hosted on huggingface.co.
            - a path to a *directory* containing model files.
        mistral_format (`bool`, *optional*):
            Tri-state control for tokenizer backend selection:

            - `True` â€” force `MistralCommonBackend` (raises `ImportError` if `mistral-common`
              is not installed, raises `OSError` if `tekken.json` is not found).
            - `False` â€” force standard HuggingFace tokenizer.
            - `None` â€” auto-detect: selects native (`MistralCommonBackend`) when
              `mistral-common` is installed and a `tekken.json` is found, regardless of
              whether HF-format files are also present. Otherwise falls back to the
              standard HuggingFace tokenizer.
        **cache_kwargs:
            Forwarded to [`~utils.cached_file`] (e.g. `cache_dir`, `force_download`,
            `local_files_only`, `revision`, `token`).

    Returns:
        `tuple[bool, str | None]`: A tuple of `(use_mistral_format, tekken_file_path)` where
        `use_mistral_format` indicates whether `MistralCommonBackend` should be used, and
        `tekken_file_path` is the resolved path to `tekken.json` (or `None`).

    Raises:
        ImportError: If `mistral_format=True` and `mistral-common` is not installed.
        OSError: If `mistral_format=True` and `tekken.json` cannot be found.
    F)FNr6   Nr7   TzZmistral_format=True requires `mistral-common`. Install it with: pip install mistral-commonzCannot find 'z' at 'zH'. Set `mistral_format=False` to use standard HuggingFace files instead.)Úpopr   ÚImportErrorr9   r   ÚOSError)r3   r;   r8   r   s       r1   Úresolve_mistral_formatr@      s  € ðN ˜ÐÐØˆ}ð ×ÒÐ<¸dÑCÔCÐCØ×ÒÐ>ÀÑEÔEÐEà˜ÐÐÝ*Ñ,Ô,ð 	ÝØlñô ð õ
 "Ð"?ÕARÐcÐcÐVbÐcÐcˆØÐÝðXÕ 1ð Xð XÐ9Vð Xð Xð Xñô ð ð �kÐ"Ð"õ 'Ñ(Ô(ð Øˆ}åÐ;Õ=NÐ_Ð_ÐR^Ð_Ð_€KØ˜tÐ# [Ð1Ð1r:   z<s>z</s>z<pad>z<unk>)Ú	bos_tokenÚ	eos_tokenÚ	pad_tokenÚ	unk_tokenÚrawc                 ór  ‡— | d         }dŠ|                       d¦  «        }|€Et          t          dg¦  «         t          t          d‰¦  «        Št          t          j        ¦  «        }t          d„ |D ¦   «         ¦  «        }|t          t          t          |¦  «        ¦  «        ¦  «        k    rt          d|› �¦  «        ‚d	„ t          d
„ |D ¦   «         ¦  «        D ¦   «         }|                      d¦  «        }|€t          |¦  «        }ˆfd„t          t          |¦  «        |¦  «        D ¦   «         }||z   |fS )aV  Derive the expected special-token strings and special-token count from a parsed tekken.json.

    Single source of truth for the special-token layout, shared by
    `MistralConverter._parse_tekken_file` (to build the vocab) and
    `_check_tekken_vocab_unchanged` (to compare against a live tokenizer), so the two cannot
    silently drift apart.

    Args:
        raw (`dict`): A parsed tekken.json, i.e. the result of `json.load` on the file.

    Returns:
        `tuple[list[str], int]`: The special-token strings, ordered by rank (the real entries
        from *raw* followed by any `<SPECIAL_i>` filler needed to reach `num_special_tokens`),
        and the resolved `num_special_tokens`.

    Raises:
        ImportError: If *raw* has no top-level `"special_tokens"` key (old tekken format) and
            `mistral-common` is not installed.
    Úconfigz<SPECIAL_{id}>Úspecial_tokensNzmistral-commonÚSPECIAL_TOKEN_TEMPLATEc              3   ó&   K  — | ]}|d          V — ŒdS )ÚrankN© ©Ú.0Úentrys     r1   ú	<genexpr>z*_derive_tekken_specials.<locals>.<genexpr>í   s&   è è € ÐLÐL¨e˜E &œMÐLÐLÐLÐLÐLÐLr:   zGtekken.json special_tokens ranks must be contiguous starting at 0, got c                 ó   — g | ]\  }}|‘ŒS rL   rL   )rN   Ú_Ú	token_strs      r1   ú
<listcomp>z+_derive_tekken_specials.<locals>.<listcomp>ñ   s,   € ð ð ð áˆAˆyð 	ðð ð r:   c           
   3   óz   K  — | ]6}|d          t          t          |d         d|d         ¦  «        ¦  «        fV — Œ7dS )rK   rS   ÚvalueN)ÚstrÚgetattrrM   s     r1   rP   z*_derive_tekken_specials.<locals>.<genexpr>ó   s]   è è € ð #
ð #
àð �6Œ]�C¥¨¨kÔ(:¸GÀUÈ;ÔEWÑ XÔ XÑYÔYÐZð#
ð #
ð #
ð #
ð #
ð #
r:   Údefault_num_special_tokensc                 ó<   •— g | ]}‰                      |¬ ¦  «        ‘ŒS ))Úid)Úformat)rN   ÚiÚfiller_templates     €r1   rT   z+_derive_tekken_specials.<locals>.<listcomp>ý   s*   ø€ ÐlÐlÐl°q�o×,Ò,°Ð,Ñ2Ô2ÐlÐlÐlr:   )Úgetr   ÚMistralConverterrX   r   ÚlistÚDEPRECATED_SPECIAL_TOKENSÚsortedÚrangeÚlenÚ
ValueError)rE   rG   Úspecial_tokens_dictsÚdeclared_ranksÚspecial_stringsÚnum_special_tokensÚfiller_stringsr^   s          @r1   Ú_derive_tekken_specialsrl   Í   sh  ø€ ð( �Œ]€FØ&€OàŸ7š7Ð#3Ñ4Ô4ÐØÐ#åÕ*Ð-=Ð,>Ñ?Ô?Ð?Ý!¥*Ð.FÈÑXÔXˆÝ#¥JÔ$HÑIÔIÐõ ÐLÐLÐ7KÐLÑLÔLÑLÔL€NØ��e¥C¨Ñ$7Ô$7Ñ8Ô8Ñ9Ô9Ò9Ð9ÝÐsÐcqÐsÐsÑtÔtÐtðð å"ð #
ð #
à-ð#
ñ #
ô #
ñ 
ô 
ðñ ô €Oð  ŸšÐ$@ÑAÔAÐØÐ!Ý  Ñ1Ô1ÐàlÐlÐlÐl½EÅ#ÀoÑBVÔBVÐXjÑ<kÔ<kÐlÑlÔl€NØ˜^Ñ+Ð-?Ð?Ð?r:   c            
       ó¼   — e Zd ZdZddedefd„Zdededdfd„Zed	e	e
ef         dee	eef         eeeef                  f         fd
„¦   «         Zdefd„Zdefd„ZdS )r`   zPConverter from Mistral tekken BPE vocab to a HuggingFace `tokenizers.Tokenizer`.FÚ
vocab_fileÚadd_prefix_spacec                 ó2   — |                       ||¦  «         dS )aI  Parse a raw `tekken.json` file into a ready-to-use converter.

        Matches `mistral_common`'s `Tekkenizer.from_file` vocab and special-token layout.

        Args:
            vocab_file (`str`): Path to a `tekken.json` file.
            add_prefix_space (`bool`): Whether to add a leading space during tokenization.
        N)Ú_parse_tekken_file)Úselfrn   ro   Úkwargss       r1   Ú__init__zMistralConverter.__init__  s!   € ð 	×Ò 
Ð,<Ñ=Ô=Ð=Ð=Ð=r:   r   Nc                 ón  ‡— t          |d¬¦  «        5 }t          j        |¦  «        }ddd¦  «         n# 1 swxY w Y   |d         }|d         }|                     d¦  «        }t	          |¦  «        \  }Šd„ |D ¦   «         }	|d         }
|�|‰z
  }|
d|…         }
d	„ |
D ¦   «         }d
„ t          |¦  «        D ¦   «         }|                      |¦  «        \  }}ˆfd„|                     ¦   «         D ¦   «         }t          |¦  «        D ]
\  }}|||<   Œ|| _        || _	        |	| _
        || _        || _        dS )zêParse a tekken.json file and set all instance attributes.

        Args:
            vocab_file (`str`): Path to a `tekken.json` file.
            add_prefix_space (`bool`): Whether to add a leading space during tokenization.
        r   r   NrG   ÚpatternÚdefault_vocab_sizec                 ó0   — g | ]}t          |d ¬¦  «        ‘ŒS )T)Úspecial)r   )rN   rS   s     r1   rT   z7MistralConverter._parse_tekken_file.<locals>.<listcomp>  s%   € Ð$pÐ$pÐ$pÈY¥Z°	À4Ð%HÑ%HÔ%HÐ$pÐ$pÐ$pr:   Úvocabc                 óB   — g | ]}t          j        |d          ¦  «        ‘ŒS )Útoken_bytes)Úbase64Ú	b64decode)rN   Úks     r1   rT   z7MistralConverter._parse_tekken_file.<locals>.<listcomp>'  s(   € ÐOÐOÐO¸A•VÔ% a¨Ô&6Ñ7Ô7ÐOÐOÐOr:   c                 ó   — i | ]\  }}||“Œ	S rL   rL   )rN   rK   Útokens      r1   ú
<dictcomp>z7MistralConverter._parse_tekken_file.<locals>.<dictcomp>(  s   € ÐNÐNÐN©+¨$°˜% ÐNÐNÐNr:   c                 ó"   •— i | ]\  }}||‰z   “ŒS rL   rL   )rN   r   Úvrj   s      €r1   r‚   z7MistralConverter._parse_tekken_file.<locals>.<dictcomp>,  s%   ø€ ÐEÐEÐE©t¨q°!��AÐ*Ñ*ÐEÐEÐEr:   )r%   r&   r'   r_   rl   Ú	enumerateÚ_extract_mergesÚitemsrv   ro   Úadditional_special_tokensÚ_precomputed_vocabÚ_precomputed_merges)rr   rn   ro   r.   ÚuntypedrG   rv   Ú
vocab_sizeÚspecial_token_stringsrˆ   Úbpe_ranks_rawÚinner_vocab_sizeÚ	bpe_ranksÚbpe_ranks_dictrz   ÚmergesrK   rS   rj   s                     @r1   rq   z#MistralConverter._parse_tekken_file  s³  ø€ õ �* wÐ/Ñ/Ô/ð 	#°1Ý”i ‘l”lˆGð	#ð 	#ð 	#ñ 	#ô 	#ð 	#ð 	#ð 	#ð 	#ð 	#ð 	#øøøð 	#ð 	#ð 	#ð 	#ð ˜Ô"ˆØ˜Ô#ˆØ—Z’ZÐ 4Ñ5Ô5ˆ
å4KÈGÑ4TÔ4TÑ1ÐÐ1à$pÐ$pÐZoÐ$pÑ$pÔ$pÐ!ð   Ô(ˆØÐ!Ø)Ð,>Ñ>ÐØ)Ð*;Ð+;Ð*;Ô<ˆMàOÐOÀÐOÑOÔOˆ	ØNÐN½À9Ñ9MÔ9MÐNÑNÔNˆà×,Ò,¨^Ñ<Ô<‰ˆˆvàEÐEÐEÐE°u·{²{±}´}ÐEÑEÔEˆÝ(Ð)>Ñ?Ô?ð 	$ð 	$‰OˆD�)Ø#ˆE�)ÑÐàˆŒØ 0ˆÔØ)BˆÔ&Ø"'ˆÔØ#)ˆÔ Ð Ð s   “4´8»8r�   c                 ó:  ‡ ‡
‡— t          ¦   «         Š
t          dt          dt          fˆ
fd„¦   «         Ši }g }‰                      ¦   «         D ]­\  }}|| ‰|¦  «        <   t          |¦  «        dk    rŒ'g }t          dt          |¦  «        ¦  «        D ]=}|d|…         ||d…         }}|‰ v r#|‰ v r||z   ‰ v r|                     |||f¦  «         Œ>t          |ˆ fd„¬¦  «        }| 	                    |¦  «         Œ®t          |d„ ¬¦  «        }ˆfd	„|D ¦   «         }	||	fS )
zMExtract a unicode vocab and ordered BPE merge list from byte-level BPE ranks.Úbr   c                 ól   •— d                      ˆfd„|                      d¦  «        D ¦   «         ¦  «        S )NÚ c                 ó:   •— g | ]}‰t          |¦  «                 ‘ŒS rL   )Úord)rN   ÚcharÚbyte_encoders     €r1   rT   zSMistralConverter._extract_merges.<locals>.token_bytes_to_string.<locals>.<listcomp>=  s$   ø€ ÐTÐTÐT¸˜L­¨T©¬Ô3ÐTÐTÐTr:   zlatin-1)ÚjoinÚdecode)r”   rš   s    €r1   Útoken_bytes_to_stringz?MistralConverter._extract_merges.<locals>.token_bytes_to_string;  s6   ø€ à—7’7ÐTÐTÐTÐTÀÇÂÈÑ@SÔ@SÐTÑTÔTÑUÔUÐUr:   r   Nc                 ó<   •— ‰| d                  ‰| d                  fS )Nr   r   rL   )Úxr�   s    €r1   ú<lambda>z2MistralConverter._extract_merges.<locals>.<lambda>K  s   ø€ °¸1¸Q¼4´À)ÈAÈaÌDÄ/Ð0R€ r:   )Úkeyc                 ó   — | d         S )Né   rL   )Úvals    r1   r    z2MistralConverter._extract_merges.<locals>.<lambda>N  s
   € ¸¸A¼€ r:   c                 óT   •— g | ]$} ‰|d          ¦  «         ‰|d         ¦  «        f‘Œ%S )r   r   rL   )rN   r¤   r�   s     €r1   rT   z4MistralConverter._extract_merges.<locals>.<listcomp>O  s?   ø€ ÐgÐgÐgÐUXÐ(Ð(¨¨Q¬Ñ0Ô0Ð2GÐ2GÈÈAÌÑ2OÔ2OÐPÐgÐgÐgr:   )
r   r   ÚbytesrW   r‡   re   rd   Úappendrc   Úextend)r�   rz   Ú
all_mergesr�   rK   ÚlocalÚindexÚpiece_lÚpiece_rr’   rš   r�   s   `         @@r1   r†   z MistralConverter._extract_merges6  s‹  øøø€ õ (Ñ)Ô)ˆå	ð	V¥Uð 	V­sð 	Vð 	Vð 	Vð 	Vð 	Vñ 
Œð	Vð !#ˆØ57ˆ
à$Ÿ?š?Ñ,Ô,ð 
	%ð 
	%‰KˆE�4Ø26ˆEÐ'Ð'¨Ñ.Ô.Ñ/Ý�5‰zŒz˜QŠˆØØˆEÝ˜q¥# e¡*¤*Ñ-Ô-ð ;ð ;�Ø#(¨¨%¨¤=°%¸¸¸´-˜�Ø˜iÐ'Ð'¨G°yÐ,@Ð,@ÀgÐPWÑFWÐ\eÐEeÐEeØ—L’L '¨7°DÐ!9Ñ:Ô:Ð:øÝ˜5Ð&RÐ&RÐ&RÐ&RÐSÑSÔSˆEØ×Ò˜eÑ$Ô$Ð$Ð$å˜JÐ,>Ð,>Ð?Ñ?Ô?ˆ
ØgÐgÐgÐgÐ\fÐgÑgÔgˆØ�fˆ}Ðr:   c                 óš   — t          t          | j        | j        d¬¦  «        ¦  «        }t	          |j        d¦  «        rd|j        _        |S )zKBuild a raw `tokenizers.Tokenizer` with BPE model (no pre/post-processing).F)Úfuse_unkÚignore_mergesT)r   r   r‰   rŠ   ÚhasattrÚmodelr°   ©rr   Ú	tokenizers     r1   r´   zMistralConverter.tokenizerR  sI   € å�c $Ô"9¸4Ô;SÐ^cÐdÑdÔdÑeÔeˆ	Ý�9”? OÑ4Ô4ð 	1Ø,0ˆIŒOÔ)ØÐr:   c                 ó|  — |                       ¦   «         }t          j        t          j        t	          | j        ¦  «        dd¬¦  «        t          j        | j        d¬¦  «        g¦  «        |_        t          j        ¦   «         |_
        |                     | j        ¦  «         t          j        d¬¦  «        |_        |S )zOBuild a fully configured `tokenizers.Tokenizer` with pre-tokenizer and decoder.ÚisolatedF)ÚbehaviorÚinvert)ro   Ú	use_regex)Útrim_offsets)r´   r
   ÚSequenceÚSplitr   rv   Ú	ByteLevelro   Úpre_tokenizerr	   ÚdecoderÚadd_special_tokensrˆ   r   Úpost_processorr³   s     r1   Ú	convertedzMistralConverter.convertedY  s¨   € à—N’NÑ$Ô$ˆ	Ý"0Ô"9åÔ$¥U¨4¬<Ñ%8Ô%8À:ÐV[Ð\Ñ\Ô\ÝÔ(¸$Ô:OÐ[`ÐaÑaÔaðñ#
ô #
ˆ	Ôõ %Ô.Ñ0Ô0ˆ	ÔØ×$Ò$ TÔ%CÑDÔDÐDå#-Ô#7ÀUÐ#KÑ#KÔ#Kˆ	Ô àÐr:   )F)Ú__name__Ú
__module__Ú__qualname__Ú__doc__rW   Úboolrt   rq   ÚstaticmethodÚdictr¦   ÚintÚtuplera   r†   r   r´   rÂ   rL   r:   r1   r`   r`     sü   € € € € € ØZÐZð	>ð 	> 3ð 	>¸$ð 	>ð 	>ð 	>ð 	>ð%*¨Sð %*ÀDð %*ÈTð %*ð %*ð %*ð %*ðN ð 4¨¨s¨
Ô#3ð ¸¸dÀ3ÈÀ8¼nÈdÐSXÐY\Ð^aÐYaÔSbÔNcÐ>cÔ8dð ð ð ñ „\ðð6˜9ð ð ð ð ð˜9ð ð ð ð ð ð r:   r`   r´   c                 ó   — | j                              d¦  «        pt          | dd¦  «        }t          |t          ¦  «        r.t          |¦  «        rt          j                             |¦  «        st          d¦  «        ‚|S )a/  Locate the on-disk `tekken.json` a tokenizer was built from.

    Called once by `save_tekken_format`, which passes the resolved path on to
    `_check_tekken_vocab_unchanged` to know what to diff against.

    Args:
        tokenizer (`TokenizersBackend`): The tokenizer to resolve a source file for. Must
            have been loaded or converted from a native Mistral tekken vocabulary file,
            i.e. its `vocab_file` must still point at an existing file whose name identifies
            it as tekken (see `is_tekken_vocab_filename`).

    Returns:
        `str`: The validated path to the tokenizer's source tekken vocabulary file.

    Raises:
        OSError: If the tokenizer's `vocab_file` does not resolve to an existing file
            identified as tekken (see `is_tekken_vocab_filename`).
    rn   Nz½Cannot save in 'mistral' format: the original tekken.json is not available. Load a native checkpoint (that still contains tekken.json) or install mistral-common to use MistralCommonBackend.)
Úinit_kwargsr_   rX   Ú
isinstancerW   r   ÚosÚpathÚisfiler?   )r´   rn   s     r1   Ú_resolve_tekken_sourcerÒ   j  s‡   € ð& Ô&×*Ò*¨<Ñ8Ô8Ðb½GÀIÈ|Ð]aÑ<bÔ<b€JÝ�z¥3Ñ'Ô'ð 
Õ,DÀZÑ,PÔ,Pð 
ÕUWÔU\×UcÒUcÐdnÑUoÔUoð 
Ýð:ñ
ô 
ð 	
ð
 Ðr:   rn   c                 ó6  — t          |d¬¦  «        5 }t          j        |¦  «        }ddd¦  «         n# 1 swxY w Y   |d         }t          |¦  «        \  }}t	          |¦  «        }|                     d¦  «        }|€t          |d         ¦  «        |z   }t          | ¦  «        }	d„ | j                             ¦   «         D ¦   «         }
t          |
|z
  ¦  «        }t          ||
z
  ¦  «        }|	|k    }|s|r‹dg}|r| 
                    d	|› �¦  «         |r| 
                    d
|› �¦  «         |r| 
                    d|	› d|› �¦  «         | 
                    d¦  «         t          d                     |¦  «        ¦  «        ‚|rt          d|	› d|› d�¦  «        ‚dS )a7  Compare *tokenizer*'s vocab against the `tekken.json` it was converted from.

    `save_format="mistral"` writes a native `tekken.json` by copying the source file
    directly, so any in-session mutation (`add_tokens`, `add_special_tokens`) would be
    silently dropped, and `tekken.json` cannot represent arbitrary added tokens anyway.
    This guard is called only from `save_tekken_format`, right before that lossy copy, to
    turn a silent drop into a raised error.

    Only two things are compared: the total vocab size, and the *set* of added-token
    content strings (via `tokenizer.added_tokens_decoder`). This does not check token ids
    or other `AddedToken` attributes (e.g. `lstrip`, `rstrip`, `normalized`).

    Args:
        tokenizer (`TokenizersBackend`): The tokenizer to check. Never mutated.
        vocab_file (`str`): Path to the source `tekken.json` to compare against.

    Raises:
        ValueError: If the vocab size or the added-token set diverges from the source
            `tekken.json`.
        OSError: If `vocab_file` cannot be opened.
    r   r   NrG   rw   rz   c                 ó   — h | ]	}|j         ’Œ
S rL   )Úcontent)rN   r�   s     r1   ú	<setcomp>z0_check_tekken_vocab_unchanged.<locals>.<setcomp>©  s   € Ð`Ð`Ð`¨u˜Uœ]Ð`Ð`Ð`r:   z“Cannot save in 'mistral' format: the tokenizer vocab state has diverged from its source tekken.json, which cannot represent arbitrary added tokens.zUnexpected added tokens: z!Missing expected special tokens: zVocab size: observed=z, expected=z7Use save_format='hf' instead to preserve these changes.ú
zGCannot save in 'mistral' format: the tokenizer reports a vocab size of z&, but its source tekken.json declares a   (via default_vocab_size/default_num_special_tokens). Either the vocab was resized in this session, in which case use save_format='hf' to preserve the change, or the source tekken.json is itself inconsistent and should be regenerated with matching vocab and config sizes.)r%   r&   r'   rl   Úsetr_   re   Úadded_tokens_decoderÚvaluesrc   r§   rf   r›   )r´   rn   r.   rE   rG   Úexpected_special_tokens_listrj   Úexpected_special_tokensÚexpected_vocab_sizeÚobserved_vocab_sizeÚobserved_added_tokensÚunexpected_tokensÚmissing_tokensÚsize_divergesÚliness                  r1   Ú_check_tekken_vocab_unchangedrä   ‡  sB  € õ, 
ˆj 7Ð	+Ñ	+Ô	+ð ¨qÝŒi˜‰lŒlˆðð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð ð �Œ]€FÝ7NÈsÑ7SÔ7SÑ4Ð Ð"4Ý!Ð">Ñ?Ô?Ðà Ÿ*š*Ð%9Ñ:Ô:ÐØÐ"Ý! # g¤,Ñ/Ô/Ð2DÑDÐå˜i™.œ.ÐØ`Ð`¸	Ô8V×8]Ò8]Ñ8_Ô8_Ð`Ñ`Ô`ÐåÐ4Ð7NÑNÑOÔOÐÝÐ3Ð6KÑKÑLÔL€NØ'Ð+>Ò>€Màð +˜Nð +ðJð
ˆð ð 	JØ�LŠLÐHÐ5FÐHÐHÑIÔIÐIØð 	OØ�LŠLÐM¸^ÐMÐMÑNÔNÐNØð 	hØ�LŠLÐfÐ1DÐfÐfÐQdÐfÐfÑgÔgÐgØ�ŠÐNÑOÔOÐOÝ˜Ÿš 5Ñ)Ô)Ñ*Ô*Ð*àð 
Ýð&Ø"ð&ð &ØJ]ð&ð &ð &ñ
ô 
ð 	
ð
ð 
s   ’3³7º7Úsave_directory.c                 óZ  — t          | ¦  «        }t          | |¦  «         t          j        |d¬¦  «         t          j                             |t          ¦  «        }t          j                             |¦  «        r t          j                             ||¦  «        st          ||¦  «         |fS )a7  Write a tokenizer back out in native Mistral format by copying its source tekken file.

    Used by `TokenizersBackend.save_pretrained` for `save_format="mistral"`. The tokenizer's
    original tekken vocabulary file (resolved via `vocab_file`) is copied byte-for-byte into
    *save_directory* as `tekken.json`, regardless of the source's own basename (e.g. a source
    named `tekken_240911.json` is normalized to `tekken.json` on save, so the output directory
    stays discoverable by both `resolve_mistral_format` and mistral-common.

    Resaving into the same directory the tokenizer was loaded from is a no-op copy: the
    source and destination are the same file, so the copy is skipped instead of raising
    `shutil.SameFileError`. The divergence guard still runs first, so an in-place resave of
    a mutated tokenizer is still rejected.

    Args:
        tokenizer (`TokenizersBackend`): The tokenizer to save. Must have been loaded or
            converted from a native Mistral tekken vocabulary file, i.e. its `vocab_file`
            must still point at an existing file identified as tekken (see
            `is_tekken_vocab_filename`), and it must not have diverged from that source
            file (see `_check_tekken_vocab_unchanged`).
        save_directory (`str` or `os.PathLike`): The path to a directory where the tekken
            vocabulary file will be written.

    Returns:
        `tuple[str, ...]`: A 1-tuple containing the path of the saved tekken vocabulary file.

    Raises:
        OSError: If the tokenizer's `vocab_file` does not resolve to an existing file
            identified as tekken (see `is_tekken_vocab_filename`).
        ValueError: If the tokenizer has diverged from its source tekken vocabulary file
            (added tokens, changed vocab size). See `_check_tekken_vocab_unchanged`.
    T)Úexist_ok)
rÒ   rä   rÏ   ÚmakedirsrÐ   r›   r   ÚexistsÚsamefiler   )r´   rå   rn   Údests       r1   Úsave_tekken_formatrì   È  s•   € õ@ (¨	Ñ2Ô2€JÝ! )¨ZÑ8Ô8Ð8å„K�¨Ð.Ñ.Ô.Ð.õ Œ7�<Š<˜Õ(9Ñ:Ô:€DÝŒG�NŠN˜4Ñ Ô ð #¥R¤W×%5Ò%5°jÀ$Ñ%GÔ%Gð #Ý�˜TÑ"Ô"Ð"Øˆ7€Nr:   Útokenizer_filec                 ó–   — t          | |¦  «        }t          | d¬¦  «        }t          d|                     ¦   «         | |dœt          ¤Ž}|S )u:  Build a `TokenizersBackend` from a Mistral `tekken.json` file.

    The chat template is resolved via a fixed precedence order (see
    `_resolve_chat_template`): explicit `chat_template` argument â†’ sibling
    `chat_template.jinja` â†’ sibling `chat_template.json` â†’ auto-generation via
    `mistral-common` â†’ `None`.

    Args:
        tokenizer_file (`str`): Path to the `tekken.json` vocabulary file.
        chat_template (`str`, *optional*): Explicit Jinja2 chat template string.
            When not provided (`None`), the template is resolved automatically
            from sibling files or generated via `mistral-common` if available.

    Returns:
        Configured `TokenizersBackend` with BPE model, special token mappings, and
        an attached chat template (or `None` when none could be resolved).
    F)rn   ro   )Útokenizer_objectrn   r   rL   )r2   r`   r   rÂ   Ú_MAP_SPECIALS)rí   r   ÚresolvedÚ	converterÚfasts        r1   Úconvert_tekken_tokenizerrô   õ  sd   € õ* & n°mÑDÔD€HÝ ¨NÈUÐSÑSÔS€IÝð Ø"×,Ò,Ñ.Ô.Ø!Øðð õ ð	ð €Dð €Kr:   )N)9rÆ   r}   r&   rÏ   Ú	functoolsr   Úpathlibr   Úshutilr   Ú
tokenizersr   r   r   r	   r
   r   Útokenizers.modelsr   Úconvert_slow_tokenizerr   Útokenization_utils_tokenizersr   Úutilsr   r   r   Ú	utils.hubr   r   Úutils.import_utilsr   Ú	constantsr   r   Ú'mistral_common.tokens.tokenizers.tekkenr   Ú
get_loggerrÃ   r*   rW   ÚPathLiker2   r9   rÇ   rË   r@   rð   rÉ   ra   rÊ   rl   r`   rÒ   rä   rì   rô   rL   r:   r1   ú<module>r     s„  ðð VÐ Uà €€€Ø €€€Ø 	€	€	€	Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YØ !Ð !Ð !Ð !Ð !Ð !à 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø >Ð >Ð >Ð >Ð >Ð >Ø <Ð <Ð <Ð <Ð <Ð <Ð <Ð <Ð <Ð <Ø PÐ PÐ PÐ PÐ PÐ PÐ PÐ PØ =Ð =Ð =Ð =Ð =Ð =Ø BÐ BÐ BÐ BÐ BÐ BÐ BÐ Bð ÐÑ Ô ð CØBÐBÐBÐBÐBÐBð 
ˆÔ	˜HÑ	%Ô	%€ð7¨¨b¬kÑ(9ð 7È#ÐPTÉ*ð 7ÐY\Ð_cÑYcð 7ð 7ð 7ð 7ðtØ#&¨¬Ñ#4ðàðð 	ˆ4�Zð	ð ð ð ð: #'ðC2ð C2Ø#&¨¬Ñ#4ðC2à˜4‘KðC2ð ˆ4��t‘ÐÔð	C2ð C2ð C2ð C2ðN ØØØð	ð €ð1@ ð 1@¨%°°S´	¸3°Ô*?ð 1@ð 1@ð 1@ð 1@ðhfð fð fð fð fñ fô fð fðRÐ&7ð ¸Cð ð ð ð ð:>
Ð->ð >
ÈCð >
ÐTXð >
ð >
ð >
ð >
ðB*Ð"3ð *ÀSÈ2Ì;ÑEVð *Ð[`ÐadÐfiÐaiÔ[jð *ð *ð *ð *ð^ !%ðð Øðà˜‘:ðð ðð ð ð ð ð r:   