§
    9²¸j[1  ã                   ó   — d Z ddlmZ ddlmZ ddlmZmZ ddlm	Z	m
Z
mZ dZdZd	Z ej        e¦  «        Z e
¦   «         rdd
lmZ ndZdZd„ Zd„ Zd„ Z G d„ ded¬¦  «        Ze	 G d„ de¦  «        ¦   «         ZdgZdS )z
Processor class for SmolVLM.
é    )Ú	timedeltaé   )Úmake_nested_list_of_images)ÚProcessingKwargsÚProcessorMixin)Úauto_docstringÚis_num2words_availableÚloggingzgYou are provided the following series of {frame_count} frames from a {video_duration} [H:MM:SS] video.
z

z
Frame from {timestamp}:)Ú	num2wordsNaÄ  <|im_start|>{% for message in messages %}{{message['role'] | capitalize}}{% if message['content'][0]['type'] == 'image' %}{{':'}}{% else %}{{': '}}{% endif %}{% for line in message['content'] %}{% if line['type'] == 'text' %}{{line['text']}}{% elif line['type'] == 'image' %}{{ '<image>' }}{% elif line['type'] == 'video' %}{{ '<video>' }}{% endif %}{% endfor %}<end_of_utterance>
{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}c           	      óÄ   — d}t          |¦  «        D ]6}t          |¦  «        D ]}||› d|dz   › d|dz   › d�z   |› | z  z   z  }Œ |dz  }Œ7|d|› �|› z   |› | z  z   |› z   z  }|S )zKPrompt with expanded image tokens for when the image is split into patches.Ú z<row_é   Ú_col_ú>ú
)Úrange)	Úimage_seq_lenÚ
image_rowsÚ
image_colsÚfake_token_around_imageÚimage_tokenÚglobal_image_tokenÚtext_split_imagesÚn_hÚn_ws	            ú~/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/smolvlm/processing_smolvlm.pyÚ_prompt_split_imager   /   sÓ   € ð ÐÝ�ZÑ Ô ð "ð "ˆÝ˜Ñ$Ô$ð 	ð 	ˆCØØ*Ð,Ð/O°s¸Q±wÐ/OÐ/OÀSÈ1ÁWÐ/OÐ/OÐ/OÑOÐU`ÐRbÐerÑRrÑrñÐÐð 	˜TÑ!ÐÐàØ&Ð$Ð&Ð&ØÐ
!ñ	"àÐ
˜]Ñ
*ñ	+ð %Ð
&ñ	'ñÐð Ðó    c                 ó&   — |› |› z   |› | z  z   |› z   S )z5Prompt with expanded image tokens for a single image.© )r   r   r   r   s       r   Ú_prompt_single_imager!   D   s7   € ð #Ð$ØÐ
!ñ	"àÐ
˜]Ñ
*ñ	+ð %Ð
&ñ	'ðr   c                 óh   — | dk    r|dk    rt          ||||¬¦  «        S t          || ||||¦  «        S )Nr   ©r   r   r   )r!   r   )r   r   r   r   r   r   s         r   Úget_image_prompt_stringr$   N   s[   € ð �Q‚€˜:¨š?˜?Ý#ØØ$;Ø#Ø1ð	
ñ 
ô 
ð 	
õ Ø�z :Ð/FÈÐUgñô ð r   c                   ó*   — e Zd ZddddœddiddidœZdS )ÚSmolVLMProcessorKwargsTF)Úadd_special_tokensÚpaddingÚis_split_into_wordsÚreturn_row_col_infoÚreturn_metadata)Útext_kwargsÚimages_kwargsÚvideos_kwargsN)Ú__name__Ú
__module__Ú__qualname__Ú	_defaultsr    r   r   r&   r&   ]   sG   € € € € € ð #'ØØ#(ð
ð 
ð " 4ð
ð ˜tð
ðð €I€I€Ir   r&   F)Útotalc            	       ó  ‡ — e Zd ZeZ	 	 ddededz  fˆ fd„Zdˆ fd„	Zdˆ fd„	Z	de
d	ed
efd„Zde
ded
efd„Z	 	 ddee
eef                  eee
eef                           z  dedz  de
dz  d
efˆ fd„Zed
ee         fd„¦   «         Zˆ xZS )ÚSmolVLMProcessoré©   Nr   Úchat_templatec                 ó˜  •— t          |dd¦  «        | _        t          |dd¦  «        | _        |                     | j        ¦  «        | _        t          |dd¦  «        | _        t          |dd¦  «        | _        || _        t          |d	d
¦  «        | _        t          st          d¦  «        ‚ t          ¦   «         j        |||fd|i|¤Ž dS )aŸ  
        image_seq_len (`int`, *optional*, defaults to 169):
            The length of the image sequence i.e. the number of <image> tokens per image in the input.
            This parameter is used to build the string from the input prompt and image tokens and should match the
            value the model used. It is computed as: image_seq_len = int(((image_size // patch_size) ** 2) / (scale_factor**2))
        Úfake_image_tokenz<fake_token_around_image>r   z<image>Úend_of_utterance_tokenz<end_of_utterance>r   z<global-img>Úvideo_tokenz<video>zbPackage `num2words` is required to run SmolVLM processor. Install it with `pip install num2words`.r7   N)Úgetattrr9   r   Úconvert_tokens_to_idsÚimage_token_idr:   r   r   r;   r   ÚImportErrorÚsuperÚ__init__)ÚselfÚimage_processorÚ	tokenizerÚvideo_processorr   r7   ÚkwargsÚ	__class__s          €r   rA   zSmolVLMProcessor.__init__q   sÞ   ø€ õ !(¨	Ð3EÐGbÑ cÔ cˆÔÝ" 9¨m¸YÑGÔGˆÔØ'×=Ò=¸dÔ>NÑOÔOˆÔÝ&-¨iÐ9QÐSgÑ&hÔ&hˆÔ#Ý")¨)Ð5IÈ>Ñ"ZÔ"ZˆÔØ*ˆÔÝ" 9¨m¸YÑGÔGˆÔåð 	ÝØtñô ð ð 	�‰ŒÔ˜¨)°_ÐlÐlÐTaÐlÐekÐlÐlÐlÐlÐlr   c                 ó”   •— |�)| j                              |¦  «        }t          |¦  «        } t          ¦   «         j        d|||dœ|¤ŽS )N©ÚimagesÚtextÚvideosr    )rC   Úfetch_imagesr   r@   Úprepare_inputs_layout)rB   rJ   rK   rL   rF   rG   s        €r   rN   z&SmolVLMProcessor.prepare_inputs_layout�   sS   ø€ ØÐØÔ)×6Ò6°vÑ>Ô>ˆFÝ/°Ñ7Ô7ˆFØ,�u‰wŒwÔ,Ð_°FÀÈfÐ_Ð_ÐX^Ð_Ð_Ð_r   c                 óþ  •‡ —  t          ¦   «         j        d|||dœ|¤Ž |€|€|€t          d¦  «        ‚|€|d u |d uz  rt          d¦  «        ‚|�¨t          ˆ fd„|D ¦   «         ¦  «        }|dk    r|€|€t          d|› d�¦  «        ‚|�6ˆ fd„|D ¦   «         }d	„ |D ¦   «         }||k    rt          d
|› d|› d�¦  «        ‚|�8ˆ fd„|D ¦   «         }d„ |D ¦   «         }	|	|k    rt          d|› d|	› d�¦  «        ‚d S d S d S )NrI   z5You must provide one of `text`, `images` or `videos`.z4You must specify exactly one of `images` or `videos`c              3   óL   •K  — | ]}|                      ‰j        ¦  «        V — Œd S )N©Úcountr   ©Ú.0ÚsamplerB   s     €r   ú	<genexpr>z3SmolVLMProcessor.validate_inputs.<locals>.<genexpr>œ   s2   øè è € Ð"UÐ"UÀf 6§<¢<°Ô0@Ñ#AÔ#AÐ"UÐ"UÐ"UÐ"UÐ"UÐ"Ur   r   zWe detected z4 tokens in the text but no images/videos were passedc                 óD   •— g | ]}|                      ‰j        ¦  «        ‘ŒS r    rQ   rS   s     €r   ú
<listcomp>z4SmolVLMProcessor.validate_inputs.<locals>.<listcomp>    ó(   ø€ Ð&YÐ&YÐ&YÈ& v§|¢|°DÔ4DÑ'EÔ'EÐ&YÐ&YÐ&Yr   c                 ó,   — g | ]}t          |¦  «        ‘ŒS r    ©Úlen©rT   Úsublists     r   rX   z4SmolVLMProcessor.validate_inputs.<locals>.<listcomp>¡   ó   € Ð%IÐ%IÐ%I°w¥c¨'¡l¤lÐ%IÐ%IÐ%Ir   z!The number of images in the text z and images z should be the same.c                 óD   •— g | ]}|                      ‰j        ¦  «        ‘ŒS r    )rR   r;   rS   s     €r   rX   z4SmolVLMProcessor.validate_inputs.<locals>.<listcomp>§   rY   r   c                 ó,   — g | ]}t          |¦  «        ‘ŒS r    r[   r]   s     r   rX   z4SmolVLMProcessor.validate_inputs.<locals>.<listcomp>¨   r_   r   z!The number of videos in the text z and videos r    )r@   Úvalidate_inputsÚ
ValueErrorÚsum)rB   rJ   rK   rL   rF   Ún_images_in_textÚn_images_per_sampleÚn_images_in_imagesÚn_videos_per_sampleÚn_videos_in_videosrG   s   `         €r   rb   z SmolVLMProcessor.validate_inputs•   sØ  øø€ Ø�‰ŒÔÐR v°DÀÐRÐRÈ6ÐRÐRÐRØˆ<˜F˜N¨v¨~ÝÐTÑUÔUÐUØˆ<˜f¨˜n°¸tÐ1CÑDˆ<ÝÐSÑTÔTÐTØÐÝ"Ð"UÐ"UÐ"UÐ"UÐPTÐ"UÑ"UÔ"UÑUÔUÐØ !Ò#Ð#¨¨¸6¸>Ý Ð!vÐ0@Ð!vÐ!vÐ!vÑwÔwÐwØÐ!Ø&YÐ&YÐ&YÐ&YÐTXÐ&YÑ&YÔ&YÐ#Ø%IÐ%IÀ&Ð%IÑ%IÔ%IÐ"Ø%Ð)<Ò<Ð<Ý$ð FÐ<Oð  Fð  FÐ]oð  Fð  Fð  Fñô ð ð Ð!Ø&YÐ&YÐ&YÐ&YÐTXÐ&YÑ&YÔ&YÐ#Ø%IÐ%IÀ&Ð%IÑ%IÔ%IÐ"Ø%Ð)<Ò<Ð<Ý$ð FÐ<Oð  Fð  FÐ]oð  Fð  Fð  Fñô ð ð Ðð "Ð!ð =Ð<r   Úimage_inputsÚ	image_idxÚreturnc                 ó´   — d„ |d         D ¦   «         }d„ |d         D ¦   «         }t          ||         ||         | j        | j        | j        | j        ¬¦  «        S )Nc                 ó   — g | ]	}|D ]}|‘ŒŒ
S r    r    )rT   Úrow_listÚrows      r   rX   z8SmolVLMProcessor.replace_image_token.<locals>.<listcomp>¯   ó%   € ÐMÐMÐM˜ÀHÐMÐM¸S�ÐMÐMÐMÐMr   Úrowsc                 ó   — g | ]	}|D ]}|‘ŒŒ
S r    r    )rT   Úcol_listÚcols      r   rX   z8SmolVLMProcessor.replace_image_token.<locals>.<listcomp>°   rq   r   Úcolsr#   )r$   r   r9   r   r   )rB   rj   rk   rF   rr   rv   s         r   Úreplace_image_tokenz$SmolVLMProcessor.replace_image_token®   so   € ØMÐM L°Ô$8ÐMÑMÔMˆØMÐM L°Ô$8ÐMÑMÔMˆÝ&Ø�ŒOØ�ŒOØÔØ$(Ô$9ØÔ(Ø#Ô6ð
ñ 
ô 
ð 	
r   Úvideo_inputsÚ	video_idxc                 óž  — |d         j         d         }|d         |         }|j        €!t                               d¦  «         d|_        d„ |j        D ¦   «         }|j        �t          |j        ¦  «        nt          |j        d         ¦  «        }t          t          |¦  «        ¬¦  «        }t           	                    t          |¦  «        t          |¦  «        ¬	¦  «        }	|D ]_}
t          | j        | j        | j        | j        ¬
¦  «        }|
d         d›d|
d         d›�}|	t"           	                    |¬¦  «        |z   z  }	Œ`|	t$          z   S )NÚpixel_valuesr   Úvideo_metadataa  SmolVLM requires frame timestamps to construct prompts, but the `fps` of the input video could not be inferred. Probably `video_metadata` was missing from inputs and you passed pre-sampled frames. Defaulting to `fps=24`. Please provide `video_metadata` for more accurate results.é   c                 óV   — g | ]&}t          |d z  ¦  «        t          |d z  ¦  «        f‘Œ'S )é<   )Úint)rT   Úseconds     r   rX   z8SmolVLMProcessor.replace_video_token.<locals>.<listcomp>Ä   s5   € Ð^Ð^Ð^À•s˜6 R™<Ñ(Ô(­#¨f°r©kÑ*:Ô*:Ð;Ð^Ð^Ð^r   éÿÿÿÿ)Úseconds)Úframe_countÚvideo_duration)r   r   r   r   Ú02dú:)Ú	timestamp)ÚshapeÚfpsÚloggerÚwarning_onceÚ
timestampsÚdurationr€   r   ÚDEFAULT_VIDEO_INTROÚformatr   Ústrr!   r   r   r9   r   ÚFRAME_TIMESTAMP_MESSAGEÚDEFAULT_MEDIA_OUTTRO)rB   rx   ry   rF   Ú
num_framesÚmetadatar�   rŽ   Úduration_tdÚpromptrˆ   Úimage_prompt_stringÚtimestamp_strs                r   Úreplace_video_tokenz$SmolVLMProcessor.replace_video_tokenº   sj  € Ø! .Ô1Ô7¸Ô:ˆ
ØÐ 0Ô1°)Ô<ˆØŒ<ÐÝ×Òðeñô ð ð
 ˆHŒLØ^Ð^È(ÔJ]Ð^Ñ^Ô^ˆ
Ø-5Ô->Ð-J•3�xÔ(Ñ)Ô)Ð)ÕPSÐT\ÔTgÐhjÔTkÑPlÔPlˆÝ­¨H©¬Ð6Ñ6Ô6ˆÝ$×+Ò+½	À*Ñ8MÔ8MÕ^aÐbmÑ^nÔ^nÐ+ÑoÔoˆØ#ð 	dð 	dˆIÝ"6ØÔ"Ø Ô,Ø(,Ô(=Ø#'Ô#:ð	#ñ #ô #Ðð  )¨œ|ÐDÐDÐD°)¸A´,ÐDÐDÐDˆMØÕ-×4Ò4¸}Ð4ÑMÔMÐPcÑcÑcˆFˆFØÕ,Ñ,Ð,r   ÚconversationÚprocessor_kwargsc                 ó@  •— t          |t          t          f¦  «        r;t          |d         t          t          f¦  «        st          |d         d¦  «        r|}n|g}t	          d„ |D ¦   «         ¦  «        }|€	|rt
          }|rA|                     d| j        j        ¦  «         |                     d| j        j	        ¦  «         n@|                     d| j        j        ¦  «         |                     d| j        j	        ¦  «          t          ¦   «         j        ||fd|i|¤ŽS )aã  
        Similar to the `apply_chat_template` method on tokenizers, this method applies a Jinja template to input
        conversations to turn them into a single tokenizable string.

        The input is expected to be in the following format, where each message content is a list consisting of text and
        optionally image or video inputs. One can also provide an image, video, URL or local path which will be used to form
        `pixel_values` when `return_dict=True`. If not provided, one will get only the formatted text, optionally tokenized text.

        conversation = [
            {
                "role": "user",
                "content": [
                    {"type": "image", "url": "https://www.ilankelman.org/stopsigns/australia.jpg"},
                    {"type": "text", "text": "Please describe this image in detail."},
                ],
            },
        ]

        Args:
            conversation (`Union[list[Dict, [str, str]], list[list[dict[str, str]]]]`):
                The conversation to format.
            chat_template (`Optional[str]`, *optional*):
                The Jinja template to use for formatting the conversation. If not provided, the tokenizer's
                chat template is used.
        r   Úcontentc              3   ó–   K  — | ]D}|D ]?}|                      d ¦  «        pg D ]%}t          |t          ¦  «        o|d         dk    V — Œ&Œ@ŒEdS )rž   ÚtypeÚvideoN)ÚgetÚ
isinstanceÚdict)rT   r›   Úmessagerž   s       r   rV   z7SmolVLMProcessor.apply_chat_template.<locals>.<genexpr>ú   s�   è è € ð 
ð 
àØ'ð
ð 
ð Ø#ŸKšK¨	Ñ2Ô2Ð8°bð	
ð 
ð õ ˜¥Ñ&Ô&ÐE¨7°6¬?¸gÒ+Eð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r   Nr”   rŠ   rœ   )r£   ÚlistÚtupleÚhasattrÚanyÚDEFAULT_CHAT_TEMPLATEÚ
setdefaultrE   r”   rŠ   r@   Úapply_chat_template)rB   r›   r7   rœ   rF   ÚconversationsÚ	has_videorG   s          €r   r¬   z$SmolVLMProcessor.apply_chat_templateÓ   s?  ø€ õ@ �l¥T­5 MÑ2Ô2ð 	+Ý�| A”­­u¨Ñ6Ô6ð	+Ý:AÀ,ÈqÄ/ÐS\Ñ:]Ô:]ð	+ð )ˆMˆMà)˜NˆMåð 
ð 
à -ð
ñ 
ô 
ñ 
ô 
ˆ	ð Ð  YÐ å1ˆMð ð 	?Ø×'Ò'¨°dÔ6JÔ6UÑVÔVÐVØ×'Ò'¨¨tÔ/CÔ/GÑHÔHÐHÐHà×Ò˜l¨DÔ,@Ô,KÑLÔLÐLØ×Ò˜e TÔ%9Ô%=Ñ>Ô>Ð>à*�u‰wŒwÔ*¨<¸ÐtÐtÐYiÐtÐmsÐtÐtÐtr   c                 ó
   — g d¢S )N)rr   rv   r|   r    )rB   s    r   Úunused_input_namesz#SmolVLMProcessor.unused_input_names  s   € à1Ð1Ð1Ð1r   )r6   N)NNN)NN)r/   r0   r1   r&   Úvalid_processor_kwargsr€   r‘   rA   rN   rb   r¤   rw   rš   r¦   r¬   Úpropertyr°   Ú__classcell__)rG   s   @r   r5   r5   m   s»  ø€ € € € € à3Ðð !Ø$(ðmð mð
 ðmð ˜T‘zðmð mð mð mð mð mð<`ð `ð `ð `ð `ð `ðð ð ð ð ð ð2

°ð 

Àð 

ÐSVð 

ð 

ð 

ð 

ð-°ð -Àð -ÐSVð -ð -ð -ð -ð8 %)Ø(,ð	9uð 9uà˜4  S œ>Ô*¨T°$°t¸CÀ¸H´~Ô2FÔ-GÑGð9uð ˜T‘zð9uð  ™+ð	9uð 
ð9uð 9uð 9uð 9uð 9uð 9uðv ð2 D¨¤Ið 2ð 2ð 2ñ „Xð2ð 2ð 2ð 2ð 2r   r5   )Ú__doc__Údatetimer   Úimage_utilsr   Úprocessing_utilsr   r   Úutilsr   r	   r
   r�   r“   r’   Ú
get_loggerr/   r‹   r   rª   r   r!   r$   r&   r5   Ú__all__r    r   r   ú<module>r»      sx  ððð ð Ð Ð Ð Ð Ð à 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ø DÐ DÐ DÐ DÐ DÐ DÐ DÐ DÐ DÐ Dð
 oð ð Ð à5Ð à	ˆÔ	˜HÑ	%Ô	%€ð ÐÑÔð Ø#Ð#Ð#Ð#Ð#Ð#Ð#à€Ið `Ð ðð ð ð*ð ð ðð ð ðð ð ð ð Ð-°Uð ñ ô ð ð  ðb2ð b2ð b2ð b2ð b2�~ñ b2ô b2ñ „ðb2ðJ Ð
€€€r   