§
    9²¸jÇ8  ã                   ól  — d dl Zd dlZd dlmZ ddlmZmZ ddl	m
Z
mZmZmZ ddlmZmZ ddlmZmZmZ ddlmZ dd	lmZmZmZ  e¦   «         rd dlmZ  ej        e¦  «        Zd
Z dZ!dZ"dZ#dZ$de%d         de%e&         fd„Z'de&de(e&e&f         fd„Z) G d„ ded¬¦  «        Z* G d„ de¦  «        Z+dgZ,dS )é    N)Ú
functionalé   )ÚBatchFeatureÚget_size_dict)ÚIMAGENET_STANDARD_MEANÚIMAGENET_STANDARD_STDÚPILImageResamplingÚSizeDict)ÚUnpackÚVideosKwargs)Ú
TensorTypeÚis_torchvision_availableÚlogging)ÚBaseVideoProcessor)ÚVideoMetadataÚgroup_videos_by_shapeÚreorder_videoszºYou are a helpful language and vision assistant. You are able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language.zgYou are provided the following series of {frame_count} frames from a {video_duration} [H:MM:SS] video.
z

z
Frame from {timestamp}:i   Úvideosútorch.TensorÚreturnc                 ó´   — t          d¦  «        x}}| D ]A}|                     ¦   «         dd…         \  }}t          ||¦  «        }t          ||¦  «        }ŒB||fS )zH
    Get the maximum height and width across all videos in a batch.
    z-inféþÿÿÿN)ÚfloatÚsizeÚmax)r   Ú
max_heightÚ	max_widthÚvideoÚheightÚwidths         ú„/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/smolvlm/video_processing_smolvlm.pyÚget_max_height_widthr"   0   sh   € õ # 6™]œ]Ð*€J�Øð *ð *ˆØŸ
š
™œ R S SÔ)‰ˆ�Ý˜ Ñ,Ô,ˆ
Ý˜˜yÑ)Ô)ˆ	ˆ	Ø˜	Ð"Ð"ó    Úresolution_max_sidec                 ó†  — |                       ¦   «         dd…         \  }}t          t          |¦  «        }|€t          ||¦  «        n|}||z  }||k    r#|}t	          ||z  ¦  «        }|dz  dk    r|dz  }n(||k    r"|}t	          ||z  ¦  «        }|dz  dk    r|dz  }t          |d¦  «        }t          |d¦  «        }||fS )a±  
    Get the output size of the video after resizing given a dictionary specifying the max and min sizes.
    Args:
        video (`np.ndarray`):
            Video to resize.
        resolution_max_side (`int`):
            The longest edge of the video will be resized to this value. The shortest edge will be resized to keep the
            input aspect ratio.
    Returns:
        The output size of the video after resizing.
    r   Né   r   é   )r   ÚminÚMAX_IMAGE_SIZEr   Úint)r   r$   r   r    Úaspect_ratios        r!   Úget_resize_output_image_sizer,   <   sæ   € ð —J’J‘L”L   Ô%�M€FˆEõ �nÐ.AÑBÔBÐØ0CÐ0K�#˜f eÑ,Ô,Ð,ÐQdÐØ˜6‘>€Là�‚€Ø#ˆÝ�U˜\Ñ)Ñ*Ô*ˆØ�A‰:˜Š?ˆ?Ø�a‰KˆFøØ	�%ŠˆØ$ˆÝ�F˜\Ñ)Ñ*Ô*ˆØ�1‰9˜Š>ˆ>Ø�Q‰JˆEå�˜‰^Œ^€FÝ��q‰MŒM€Eà�5ˆ=Ðr#   c                   ó*   — e Zd ZU eeef         ed<   dS )ÚSmolVLMVideoProcessorInitKwargsÚmax_image_sizeN)Ú__name__Ú
__module__Ú__qualname__ÚdictÚstrr*   Ú__annotations__© r#   r!   r.   r.   d   s&   € € € € € € Ø˜˜c˜”NÐ"Ð"Ñ"Ð"Ð"r#   r.   F)Útotalc                   óŽ  ‡ — e Zd Zej        ZddiZddiZeZ	e
ZdZdZdZdZdZdZeZddgZdee         fˆ fd	„Z	 	 d*dddedddeddf
ˆ fd„Z	 	 d+dddeeef         dededef
d„Z	 	 	 d,deded
z  deez  d
z  ded
z  fd„Z	 d-de d         d ed!ededdd"ed#ed$ed%ed&ee e         z  d
z  d'ee e         z  d
z  d(e!e"z  d
z  fd)„Z#ˆ xZ$S ).ÚSmolVLMVideoProcessorÚlongest_edgei°  il  TFÚpixel_valuesÚpixel_attention_maskÚkwargsc                 ó  •—  t          ¦   «         j        di |¤Ž d|v rd|v r|d         |d         d<   d|v rO|d         d         | _        |d         d         | _        t	          |d         d         | j        ¬¦  «        | _        d S d S )Nr   Úvideo_samplingÚ
video_sizeÚ
max_framesÚfps)Údefault_to_squarer6   )ÚsuperÚ__init__Ú
num_framesrB   r   rC   r   )Úselfr=   Ú	__class__s     €r!   rE   zSmolVLMVideoProcessor.__init__w   sª   ø€ Ø�‰ŒÔÐ"Ð"˜6Ð"Ð"Ð"ð �VÐÐÐ 0°FÐ :Ð :Ø5;¸F´^ˆFÐ#Ô$ \Ñ2à˜vÐ%Ð%Ø$Ð%5Ô6°|ÔDˆDŒOØÐ.Ô/°Ô6ˆDŒHÝ% fÐ-=Ô&>¸|Ô&LÐ`dÔ`vÐwÑwÔwˆDŒIˆIˆIð &Ð%r#   Nr   r   r   Úresamplez7PILImageResampling | tvF.InterpolationMode | int | NoneÚ	antialiasr   c                 ó¼  •— |j         rt          ||j         ¬¦  «        }n0|j        r|j        r|j        |j        f}nt	          d|› d�¦  «        ‚t          ¦   «                              |t          |d         |d         ¬¦  «        ||¬¦  «        }t          | j        d         | j        d         ¬¦  «        }t          ¦   «                              ||||¬¦  «        }|S )	a'  
        Resize an video to `(size["height"], size["width"])`.
        Args:
            video (`torch.Tensor`):
                Video to resize.
            size (`SizeDict`):
                Dictionary in the format `{"height": int, "width": int}` specifying the size of the output video.
            resample (`PILImageResampling` or `InterpolationMode`, *optional*, defaults to `InterpolationMode.BILINEAR`):
                Resampling filter to use when resizing the video.
        Returns:
            `torch.Tensor`: The resized video.
        )r$   zHSize must contain 'height' and 'width' keys, or 'longest_edge' key. Got ú.r   r'   )r   r    )rI   rJ   r:   )	r:   r,   r   r    Ú
ValueErrorrD   Úresizer
   r/   )	rG   r   r   rI   rJ   r=   Únew_sizeÚmax_sizerH   s	           €r!   rN   zSmolVLMVideoProcessor.resizeƒ   sö   ø€ ð( Ôð 
	qõ 4ØØ$(Ô$5ðñ ô ˆHˆHð Œ[ð 	q˜TœZð 	qØœ T¤ZÐ0ˆHˆHåÐoÐhlÐoÐoÐoÑpÔpÐpå‘”—’Ø•8 8¨A¤;°h¸q´kÐBÑBÔBÈXÐajð ñ 
ô 
ˆõ  4Ô#6°~Ô#FÈdÔNaÐbpÔNqÐrÑrÔrˆÝ‘”—’˜u h¸ÈY�ÑWÔWˆØˆr#   r   Úpadded_sizeÚmax_num_framesÚfillÚreturn_pixel_maskc                 ó$  — |                      ¦   «         dd…         }|j        dk    r|j        d         n|j        d         }|d         |d         z
  }|d         |d         z
  }	||z
  }
|	dk     s|dk     s|
dk     rt          d|› d|› d|› d	|› d
�	¦  «        ‚||k    s|
dk    r!d|	d|ddd|
g}t	          j        |||¬¦  «        }d}|rFt          j        |dddd…dd…f         t          j        ¬¦  «        }d|dd|d         …d|d         …f<   ||fS )a*  Pads the sample with empty video to the padded_size
        Args:
            video (`torch.Tensor`):
                Batched video to pad.
            padded_size (`tuple[int, int]`):
                Height and width to pad.
            max_num_frames (`int`):
                The maximum number of frames to which video will be padded.
            fill (`int`, *optional*):
                The value to use for the padding.
            return_pixel_mask (`bool`, *optional*, defaults to `True`):
                Whether to return a pixel mask.
        r   Né   r'   r   z‰Padding dimensions are negative. Please make sure that the padded size is larger than the original size. Got padded max number of frames z and padded size: z, original number of frames z and size: rL   )Úvalue.©Údtype)	r   ÚndimÚshaperM   ÚFÚpadÚtorchÚ
zeros_likeÚint64)rG   r   rQ   rR   rS   rT   Úoriginal_sizerF   Úpadding_heightÚpadding_widthÚpadding_frameÚpaddingÚ
pixel_masks                r!   r]   zSmolVLMVideoProcessor.pad­   s“  € ð* Ÿ
š
™œ R S SÔ)ˆØ',¤z°Q¢ �U”[ ”^�^¸E¼KÈ¼Nˆ
Ø$ Qœ¨-¸Ô*:Ñ:ˆØ# Aœ¨°qÔ)9Ñ9ˆØ&¨Ñ3ˆØ˜1ÒÐ °Ò 2Ð 2°mÀaÒ6GÐ6GÝðUØBPðUð UØdoðUð Uà-7ðUð UàDQðUð Uð Uñô ð ð
 ˜KÒ'Ð'¨=¸1Ò+<Ð+<Ø˜-¨¨N¸A¸qÀ!À]ÐSˆGÝ”E˜% °Ð5Ñ5Ô5ˆEð ˆ
Øð 	HÝÔ)¨%°°Q¸¸¸¸1¸1¸1°Ô*=ÅUÄ[ÐQÑQÔQˆJØFGˆJ�sÐ.˜m¨AÔ.Ð.Ð0B°-ÀÔ2BÐ0BÐBÑCà�jÐ Ð r#   r'   ÚmetadatarF   rB   Ú	skip_secsc                 ój  — |�t          |dd¦  «        €t          d¦  «        ‚|�|n| j        }|�|n| j        }|j        }t          t          ||d         z  ¦  «        ¦  «        }t          ||¦  «        }|dk     rd}d}	|dz
  }
|dk    rH|d         d|z  z
  ||z  k    r3t          ||d         z  ¦  «        }	t          |||d         z  z
  ¦  «        }
t          d|	¦  «        }	t          |
|dz
  ¦  «        }
|	|
k    rd|dz
  }
}	t          j
        |	|
|t
          ¬¦  «        }t          j        |¦  «        }|S )	a!  
        Video sampling function which:
            - Uses `num_frames` (if provided) or calculates it from `fps` and metadata.
            - Applies a basic center-skip if fewer frames than available, otherwise
                optionally skips `skip_secs` from both the start and end.
            - Uniformly samples the desired number of frames between the start and end indices.

        Args:
            metadata (`VideoMetadata`):
                Metadata of the video containing information about total duration, fps and total number of frames.
            num_frames (`int`, *optional*):
                Maximum number of frames to sample. Defaults to `self.num_frames`.
            fps (`int` or `float`, *optional*):
                Target frames to sample per second. Defaults to `self.fps`.
            skip_secs (`float`, *optional*, defaults to `1`):
                Number of seconds to skip from the start and end if the video is long enough.

        Returns:
            np.ndarray:
                Indices to sample video frames.
        NrB   z¶Asked to sample frames per second but no video metadata was provided which is required when sampling in SmolVLM. Please pass in `VideoMetadata` object or set `do_sample_frames=False`Údurationr'   r   r&   rX   )ÚgetattrrM   rF   rB   Útotal_num_framesr*   Úroundr(   r   ÚnpÚlinspaceÚunique)rG   rg   rF   rB   rh   r=   rl   Úestimated_framesÚdesired_framesÚ	start_idxÚend_idxÚindicess               r!   Úsample_framesz#SmolVLMVideoProcessor.sample_framesÚ   su  € ð: Ð�w x°¸Ñ=Ô=ÐEÝðXñô ð ð
 $.Ð#9�Z�Z¸t¼ˆ
Ø�_ˆcˆc¨$¬(ˆØ#Ô4Ðõ �u S¨8°JÔ+?Ñ%?Ñ@Ô@ÑAÔAÐõ Ð-¨zÑ:Ô:ˆØ˜AÒÐØˆNð ˆ	Ø" QÑ&ˆà�qŠ=ˆ=˜h zÔ2°Q¸±]ÑBÀzÐTWÑGWÒXÐXÝ˜I¨°¬Ñ7Ñ8Ô8ˆIÝÐ*¨Y¸À%¼Ñ-HÑHÑIÔIˆGå˜˜9Ñ%Ô%ˆ	Ý�gÐ/°!Ñ3Ñ4Ô4ˆØ˜ÒÐØ!"Ð$4°qÑ$8�wˆIå”+˜i¨°.ÍÐLÑLÔLˆÝ”)˜GÑ$Ô$ˆàˆr#   r   Údo_convert_rgbÚ	do_resizeÚ
do_rescaleÚrescale_factorÚdo_normalizeÚdo_padÚ
image_meanÚ	image_stdÚreturn_tensorsc           	      óN  — t          |¦  «        \  }}i }|                     ¦   «         D ];\  }}|r|                      |¦  «        }|r|                      |||¬¦  «        }|||<   Œ<t	          ||¦  «        }t          |¦  «        \  }}i }|                     ¦   «         D ]$\  }}|                      |||||
|¦  «        }|||<   Œ%t	          ||¦  «        }|	r�t          |¦  «        }t          d„ |D ¦   «         ¦  «        }t          |¦  «        \  }}i }i }|                     ¦   «         D ]*\  }}|                      |||¬¦  «        \  }}|||<   |||<   Œ+t	          ||¦  «        }t	          ||¦  «        }d|i}|	r|	r|�t          j
        |d¬¦  «        n||d<   t          ||¬¦  «        S )	N)r   rI   c              3   ó4   K  — | ]}t          |¦  «        V — Œd S ©N)Úlen)Ú.0r   s     r!   ú	<genexpr>z4SmolVLMVideoProcessor._preprocess.<locals>.<genexpr>A  s(   è è € Ð JÐ J°¥ U¡¤Ð JÐ JÐ JÐ JÐ JÐ Jr#   )rQ   rR   r;   r   )Údimr<   )Útensor_type)r   ÚitemsÚconvert_to_rgbrN   r   Úrescale_and_normalizer"   r   r]   r^   Ústackr   )rG   r   rw   rx   r   rI   ry   rz   r{   r|   r}   r~   r   r=   Úgrouped_videosÚgrouped_videos_indexÚresized_videos_groupedr[   Ústacked_videosÚresized_videosÚprocessed_videos_groupedÚprocessed_videosÚpad_sizerR   Úprocessed_padded_mask_groupedÚpadded_masksr<   Údatas                               r!   Ú_preprocessz!SmolVLMVideoProcessor._preprocess  sP  € õ  0EÀVÑ/LÔ/LÑ,ˆÐ,Ø!#ÐØ%3×%9Ò%9Ñ%;Ô%;ð 	;ð 	;Ñ!ˆE�>Øð EØ!%×!4Ò!4°^Ñ!DÔ!D�Øð [Ø!%§¢¨^À$ÐQY Ñ!ZÔ!Z�Ø,:Ð" 5Ñ)Ð)Ý'Ð(>Ð@TÑUÔUˆå/DÀ^Ñ/TÔ/TÑ,ˆÐ,Ø#%Ð Ø%3×%9Ò%9Ñ%;Ô%;ð 	=ð 	=Ñ!ˆE�>Ø!×7Ò7Ø 
¨N¸LÈ*ÐV_ñô ˆNð /=Ð$ UÑ+Ð+å)Ð*BÐDXÑYÔYÐàð 	gÝ+Ð,<Ñ=Ô=ˆHÝ Ð JÐ JÐ9IÐ JÑ JÔ JÑJÔJˆNÝ3HÐIYÑ3ZÔ3ZÑ0ˆNÐ0Ø,.Ð)Ø')Ð$à)7×)=Ò)=Ñ)?Ô)?ð Dð DÑ%��~Ø/3¯xªxØ"°Èð 08ñ 0ô 0Ñ,� ð 3AÐ(¨Ñ/Ø7CÐ-¨eÑ4Ð4å-Ð.FÐH\Ñ]Ô]ÐÝ#1Ð2OÐQeÑ#fÔ#fÐ àÐ 0Ð1ˆàð 	ð ð*Ø,Ð8õ ”Ð0°aÐ8Ñ8Ô8Ð8à)ð Ð'Ñ(õ
 ˜D¨nÐ=Ñ=Ô=Ð=r#   )NT)r   T)NNr'   r‚   )%r0   r1   r2   r	   ÚLANCZOSrI   r   r/   r   r}   r   r~   rx   ry   r{   rw   r|   Údo_sample_framesr.   Úvalid_kwargsÚmodel_input_namesr   rE   r
   ÚboolrN   Útupler*   r]   r   r   rv   Úlistr4   r   r—   Ú__classcell__)rH   s   @r!   r9   r9   h   s|  ø€ € € € € Ø!Ô)€HØ˜GÐ$€DØ$ cÐ*€NØ'€JØ%€IØ€IØ€JØ€LØ€NØ€FØÐØ2€LØ'Ð)?Ð@Ðð
x Ð(GÔ!Hð 
xð 
xð 
xð 
xð 
xð 
xð  OSØð(ð (àð(ð ð(ð Lð	(ð
 ð(ð 
ð(ð (ð (ð (ð (ð (ð^ Ø"&ð+!ð +!àð+!ð ˜3 ˜8”_ð+!ð ð	+!ð
 ð+!ð  ð+!ð +!ð +!ð +!ð` "&Ø"&Ø !ð?ð ?àð?ð ˜$‘Jð?ð �5‰[˜4Ñð	?ð
 ˜‘:ð?ð ?ð ?ð ?ð\ 37ð=>ð =>à�^Ô$ð=>ð ð=>ð ð	=>ð
 ð=>ð Lð=>ð ð=>ð ð=>ð ð=>ð ð=>ð ˜D œKÑ'¨$Ñ.ð=>ð ˜4 œ;Ñ&¨Ñ-ð=>ð ˜jÑ(¨4Ñ/ð=>ð =>ð =>ð =>ð =>ð =>ð =>ð =>r#   r9   )-Únumpyrn   r^   Útorch.nnr   r\   Úimage_processing_utilsr   r   Úimage_utilsr   r   r	   r
   Úprocessing_utilsr   r   Úutilsr   r   r   Úvideo_processing_utilsr   Úvideo_utilsr   r   r   Útorchvision.transforms.v2ÚtvFÚ
get_loggerr0   ÚloggerÚDEFAULT_SYSTEM_MESSAGEÚDEFAULT_VIDEO_INTROÚDEFAULT_MEDIA_OUTTROÚFRAME_TIMESTAMP_MESSAGEr)   rž   r*   r"   r�   r,   r.   r9   Ú__all__r6   r#   r!   ú<module>r±      s  ðð  Ð Ð Ð Ø €€€Ø $Ð $Ð $Ð $Ð $Ð $à AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aðð ð ð ð ð ð ð ð ð ð ð ð 5Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4Ø BÐ BÐ BÐ BÐ BÐ BÐ BÐ BÐ BÐ BØ 8Ð 8Ð 8Ð 8Ð 8Ð 8Ø OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ Oð ÐÑÔð <Ø;Ð;Ð;Ð;Ð;Ð;ð 
ˆÔ	˜HÑ	%Ô	%€ð VÐ ànð ð Ð Ø5Ð Ø€ð	#  nÔ!5ð 	#¸$¸s¼)ð 	#ð 	#ð 	#ð 	#ð%àð%ð ˆ3�ˆ8„_ð%ð %ð %ð %ðP#ð #ð #ð #ð # l¸%ð #ñ #ô #ð #ðp>ð p>ð p>ð p>ð p>Ð.ñ p>ô p>ð p>ðf #Ð
#€€€r#   