§
    9²¸j2
  ã                   ó8   — d Z ddlmZmZmZ  G d„ d¦  «        ZdS )z Tokenization utils for RoFormer.é    )ÚNormalizedStringÚPreTokenizedStringÚnormalizersc                   óB   — e Zd Zd	d„Zdededee         fd„Zdefd„Z	dS )
ÚJiebaPreTokenizerÚreturnNc                 ó¢   — || _         t          j        dddd¬¦  «        | _        	 dd l}n# t          $ r t	          d¦  «        ‚w xY w|| _        d S )NFT)Ú
clean_textÚhandle_chinese_charsÚstrip_accentsÚ	lowercaser   zkYou need to install rjieba to use RoFormerTokenizer. See https://pypi.org/project/rjieba/ for installation.)Úvocabr   ÚBertNormalizerÚrjiebaÚImportErrorÚjieba)Úselfr   r   s      ú/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/roformer/tokenization_utils.pyÚ__init__zJiebaPreTokenizer.__init__   s   € ØˆŒ
Ý&Ô5ØØ!%ØØð	
ñ 
ô 
ˆÔð	ØˆMˆMˆMˆMøÝð 	ð 	ð 	ÝðIñô ð ð	øøøð
 ˆŒ
ˆ
ˆ
s	   ¦+ «AÚiÚnormalized_stringc                 ó€  — g }| j                              t          |¦  «        d¬¦  «        D ]‘\  }}}|| j        v r|                     |||…         ¦  «         Œ-| j                             |¦  «                             ¦   «         }|D ]5}|r1|t          |¦  «        z   }|                     |||…         ¦  «         |}Œ6Œ’|S )NF)Úhmm)	r   ÚtokenizeÚstrr   Úappendr   Únormalize_strÚsplitÚlen)r   r   r   ÚsplitsÚtokenÚstartÚendÚ
token_lists           r   Újieba_splitzJiebaPreTokenizer.jieba_split%   sá   € Øˆð "&¤×!4Ò!4µSÐ9JÑ5KÔ5KÐQVÐ!4Ñ!WÔ!Wð 		$ð 		$ÑˆE�5˜#Ø˜œ
Ð"Ð"Ø—’Ð/°°c°	Ô:Ñ;Ô;Ð;Ð;à!Ô-×;Ò;¸EÑBÔB×HÒHÑJÔJ�
Ø'ð $ð $�EØð $Ø#¥c¨%¡j¤jÑ0˜ØŸšÐ&7¸¸c¸	Ô&BÑCÔCÐCØ #˜øð	$ð  ˆó    Úpretokc                 ó:   — |                      | j        ¦  «         d S )N)r   r%   )r   r'   s     r   Úpre_tokenizezJiebaPreTokenizer.pre_tokenize@   s   € Ø�Š�TÔ%Ñ&Ô&Ð&Ð&Ð&r&   )r   N)
Ú__name__Ú
__module__Ú__qualname__r   Úintr   Úlistr%   r   r)   © r&   r   r   r      st   € € € € € ðð ð ð ð"˜Sð Ð5Eð È$ÐO_ÔJ`ð ð ð ð ð6'Ð#5ð 'ð 'ð 'ð 'ð 'ð 'r&   r   N)Ú__doc__Ú
tokenizersr   r   r   r   r/   r&   r   ú<module>r2      s]   ðð 'Ð &à HÐ HÐ HÐ HÐ HÐ HÐ HÐ HÐ HÐ Hð.'ð .'ð .'ð .'ð .'ñ .'ô .'ð .'ð .'ð .'r&   