§
    9²¸j   ã                   óD   — d Z ddlmZ ddiZdZdedeeeef         fd„Zd	S )
aD  Building a tokenizer from a GGUF file's metadata.

A GGUF repo ships no `tokenizer.json`: the vocabulary, merges and special token ids are metadata keys.
The renaming table is llama.cpp's convention, shared with `integrations.ggml.GGUF_TOKENIZER_MAPPING`;
the only per-architecture fact is which converter reads the result.
é   )Úread_gguf_metadataÚqwen35Úqwen3_5_text)ztokenizer.ggml.tokensztokenizer.ggml.mergesÚ	gguf_pathÚreturnc                 ó–  ‡	— ddl m} t          | t          ¦  «        \  Š	}‰	d         }|t          vr(t          d|›dt          t          ¦  «        › d�¦  «        ‚ˆ	fd„|                     ¦   «         D ¦   «         }|d         |d	         }}d
D ]/}|                     |› d�¦  «        }|�|d         |         nd||<   Œ0t          |         ||fS )zÑ`(model_type, tokenizer_dict, tokenizer_config)` for the tokenizer this file describes.

    Raises for an architecture with no entry above; callers with a fallback check
    `GGUF_TOKENIZER_ARCHS` first.
    é   )ÚGGUF_TOKENIZER_MAPPINGzgeneral.architecturez:Cannot build a tokenizer from a GGUF file of architecture z. Supported: ú.c                 óX   •— i | ]&\  }}|ˆfd „|                      ¦   «         D ¦   «         “Œ'S )c                 ó<   •— i | ]\  }}d |› �‰v ¯|‰d |› �         “ŒS )z
tokenizer.© )Ú.0ÚkeyÚnameÚmetadatas      €ú…/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/integrations/gguf/gguf_tokenizer_mapping.pyú
<dictcomp>z1get_gguf_tokenizer.<locals>.<dictcomp>.<dictcomp>1   sI   ø€ ð 
ð 
ð 
Ù3<°3¸ÐSeÐ`cÐSeÐSeÐiqÐSqÐSqˆD�(Ð-¨Ð-Ð-Ô.ÐSqÐSqÐSqó    )Úitems)r   ÚsectionÚrenamesr   s      €r   r   z&get_gguf_tokenizer.<locals>.<dictcomp>0   s]   ø€ ð ð ð ñ ˆG�Wð 	ð 
ð 
ð 
ð 
Ø@GÇÂÁÄð
ñ 
ô 
ðð ð r   Ú	tokenizerÚtokenizer_config)Ú	bos_tokenÚ	eos_tokenÚ	pad_tokenÚ	unk_tokenÚ_idNÚtokens)	Úggmlr
   r   Ú_VOCABULARY_KEYSÚGGUF_TOKENIZER_ARCHSÚ
ValueErrorÚsortedr   Úget)
r   r
   Ú_ÚarchitectureÚsectionsr   r   r   Útoken_idr   s
            @r   Úget_gguf_tokenizerr+   !   s0  ø€ ð .Ð-Ð-Ð-Ð-Ð-å$ YÕ0@ÑAÔA�K€HˆaØÐ2Ô3€LØÕ/Ð/Ð/Ýð:Èð :ð :Ý Õ!5Ñ6Ô6ð:ð :ð :ñ
ô 
ð 	
ðð ð ð ð !7× <Ò <Ñ >Ô >ð	ñ ô €Hð #+¨;Ô"7¸ÐBTÔ9UÐ€Ið Eð að aˆØ#×'Ò'¨4¨¨¨Ñ5Ô5ˆØBJÐBV ¨8Ô!4°XÔ!>Ð!>Ð\`Ð˜ÑÐÝ Ô-¨yÐ:JÐJÐJr   N)	Ú__doc__Úreaderr   r#   r"   ÚstrÚtupleÚdictr+   r   r   r   ú<module>r1      sz   ððð ð 'Ð &Ð &Ð &Ð &Ð &ð
 ˆnðÐ ð
 FÐ ðK #ð K¨%°°T¸4°Ô*@ð Kð Kð Kð Kð Kð Kr   