§
    9²¸jÞ3  ã                   ó  — d dl Z d dlZd dlZd dlmZ ddlmZ ddl	m
Z
mZmZmZmZmZ ddlmZmZ ddlmZmZ ddlmZ dd	lmZmZmZ  G d
„ ded¬¦  «        Z	 	 	 	 ddededededededefd„Ze G d„ de¦  «        ¦   «         ZdgZ dS )é    N)Ú
functionalé   )ÚBatchFeature)ÚOPENAI_CLIP_MEANÚOPENAI_CLIP_STDÚChannelDimensionÚPILImageResamplingÚSizeDictÚget_image_size)ÚUnpackÚVideosKwargs)Ú
TensorTypeÚauto_docstring)ÚBaseVideoProcessor)ÚVideoMetadataÚgroup_videos_by_shapeÚreorder_videosc                   ó`   — e Zd ZU dZeeef         ed<   eed<   eed<   eed<   eed<   eed<   dS )	ÚGlmgaVideoProcessorInitKwargsaÐ  
    patch_size (`int`, *optional*, defaults to 14):
        The spatial patch size of the vision encoder.
    temporal_patch_size (`int`, *optional*, defaults to 2):
        The temporal patch size of the vision encoder.
    merge_size (`int`, *optional*, defaults to 2):
        The merge size of the vision encoder to llm encoder.
    max_frames (`int`, *optional*, defaults to 640):
        The maximum number of frames that can be sampled.
    max_image_size (`dict`, *optional*, defaults to `28 * 28 * 2 * 55790`):
        The maximum pixels a video can be resized to.
    patch_expand_factor (`int`, *optional*, defaults to 1):
        The factor that will be multiplied to `patch_size` when resizing videos.
    Úmax_image_sizeÚ
patch_sizeÚtemporal_patch_sizeÚ
merge_sizeÚpatch_expand_factorÚ
max_framesN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚdictÚstrÚintÚ__annotations__© ó    ú€/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/glmga/video_processing_glmga.pyr   r   *   sc   € € € € € € ðð ð ˜˜c˜”NÐ"Ð"Ñ"Ø€O€O�OØÐÐÑØ€O€O�OØÐÐÑØ€O€O�O€O€Or%   r   F)Útotalé   é   é 1  é  “ Ú
num_framesÚheightÚwidthÚtemporal_factorÚfactorÚ
min_pixelsÚ
max_pixelsc                 ó¼  — | |k     rt          d| › d|› �¦  «        ‚||k     s||k     r:t          ||z  ||z  ¦  «        }t          ||z  ¦  «        }t          ||z  ¦  «        }t          ||¦  «        t          ||¦  «        z  dk    r1t          dt          ||¦  «        t          ||¦  «        z  › �¦  «        ‚t	          ||z  ¦  «        |z  }t	          ||z  ¦  «        |z  }	t	          | |z  ¦  «        |z  }
|
|z  |	z  |k    rtt          j        | |z  |z  |z  ¦  «        }t          |t          j        ||z  |z  ¦  «        |z  ¦  «        }t          |t          j        ||z  |z  ¦  «        |z  ¦  «        }	nc|
|z  |	z  |k     rWt          j        || |z  |z  z  ¦  «        }t          j        ||z  |z  ¦  «        |z  }t          j        ||z  |z  ¦  «        |z  }	||	fS )Nzt:z% must be larger than temporal_factor:éÈ   z4absolute aspect ratio must be smaller than 200, got )	Ú
ValueErrorÚmaxr"   ÚminÚroundÚmathÚsqrtÚfloorÚceil)r,   r-   r.   r/   r0   r1   r2   ÚscaleÚh_barÚw_barÚt_barÚbetas               r&   Úsmart_resizerB   B   s  € ð �OÒ#Ð#ÝÐ`˜jÐ`Ð`ÈÐ`Ð`ÑaÔaÐaØ�‚€˜% &š.˜.Ý�F˜V‘O V¨e¡^Ñ4Ô4ˆÝ�V˜e‘^Ñ$Ô$ˆÝ�E˜E‘MÑ"Ô"ˆå
ˆ6�5ÑÔ�C ¨Ñ.Ô.Ñ.°Ò4Ð4ÝØlÅ3ÀvÈuÑCUÔCUÕX[Ð\bÐdiÑXjÔXjÑCjÐlÐlñ
ô 
ð 	
õ �&˜6‘/Ñ"Ô" VÑ+€EÝ�%˜&‘.Ñ!Ô! FÑ*€EÝ�*˜Ñ.Ñ/Ô/°/ÑA€Eàˆu�}�uÑ˜zÒ)Ð)ÝŒy˜* vÑ-°Ñ5¸ÑCÑDÔDˆÝ�F�DœJ v°¡}°vÑ'=Ñ>Ô>ÀÑGÑHÔHˆÝ�F�DœJ u¨t¡|°fÑ'<Ñ=Ô=ÀÑFÑGÔGˆˆØ	�‰˜Ñ	 Ò	+Ð	+ÝŒy˜ z°FÑ':¸UÑ'BÑCÑDÔDˆÝ”	˜& 4™-¨&Ñ0Ñ1Ô1°FÑ:ˆÝ”	˜% $™,¨Ñ/Ñ0Ô0°6Ñ9ˆà�%ˆ<Ðr%   c                    óæ  ‡ — e Zd Zej        ZdddœZddiZeZ	e
ZdZdZdZdZdZdZdZdZdZeZd	ZdZd
dgZdZdZdee         fˆ fd„Z	 d,dedee z  dz  fd„Z!ddde"dddededdfˆ fd„Z#dddededede$deef         f
d „Z%dddej        dd!ddddddddfde&e'j(                 d"e)d#e)de"dz  ddd$e)d%e d&e)d'e e&e          z  dz  d(e e&e          z  dz  d)edz  dedz  dedz  dedz  d*e*e+z  dz  fd+„Z,ˆ xZ-S )-ÚGlmgaVideoProcessorr*   iÀÑ6)Úshortest_edgeÚlongest_edgerF   Té   r(   i,  é   Úpixel_values_videosÚvideo_grid_thwé   i€  Úkwargsc                 ó:   •—  t          ¦   «         j        di |¤Ž d S )Nr$   )ÚsuperÚ__init__)ÚselfrL   Ú	__class__s     €r&   rO   zGlmgaVideoProcessor.__init__~   s&   ø€ Ø�‰ŒÔÐ"Ð"˜6Ð"Ð"Ð"Ð"Ð"r%   NÚmetadataÚfpsc                 óÊ  ‡‡‡— |�t          |dd¦  «        €t          d¦  «        ‚|j        Š‰dz
  }|j        pt	          ||j        z  ¦  «        dz   }|�|n| j        }t          ||z  ¦  «        Št          ‰| j        ¦  «        Šd|j        z  Šˆfd„t          ‰¦  «        D ¦   «         }‰‰k     rˆˆfd„t          ‰¦  «        D ¦   «         }nLg }d}	d|z  }
t          ‰¦  «        D ]3}||         |	k    r%|	|
z  }	| 
                    |¦  «         |	||
z
  k    r nŒ4t          |¦  «        ‰k     rit          |¦  «        dk    rdt          ‰dz
  d¦  «        }}n|d         |d         }}t          j        ||‰t          ¬	¦  «                             ¦   «         }nEt          |¦  «        ‰k    r2t          j        d‰dz
  ‰t          ¬	¦  «                             ¦   «         }t!          ¦   «         g }}|D ]0}||vr*|                     |¦  «         | 
                    |¦  «         Œ1t          |¦  «        dz  r| 
                    |d         ¦  «         t          j        |t          ¬	¦  «        S )
a‡  
        Args:
            metadata (`VideoMetadata`):
                Metadata of the video containing information about total duration, fps and total number of frames.
            fps (`int` or `float`, *optional*):
                Target frames to sample per second. Defaults to `self.fps`.
        Returns:
            np.ndarray:
                Indices to sample video frames.
        NrS   z´Asked to sample frames per second but no video metadata was provided which is required when sampling in Glmga. Please pass in `VideoMetadata` object or set `do_sample_frames=False`rK   c                 ó   •— g | ]}|‰z  ‘ŒS r$   r$   )Ú.0ÚiÚduration_per_frames     €r&   ú
<listcomp>z5GlmgaVideoProcessor.sample_frames.<locals>.<listcomp>¡   s   ø€ ÐJÐJÐJ°�aÐ,Ñ,ÐJÐJÐJr%   c                 óD   •— g | ]}t          j        |‰z  ‰z  ¦  «        ‘ŒS r$   )r9   r;   )rV   Ú_iÚ	extract_tÚtotal_framess     €€r&   rY   z5GlmgaVideoProcessor.sample_frames.<locals>.<listcomp>¤   s-   ø€ ÐbÐbÐbÈ2�TœZ¨¨\Ñ(9¸IÑ(EÑFÔFÐbÐbÐbr%   r   éÿÿÿÿ)Údtype)Úgetattrr5   Útotal_num_framesÚdurationr8   rS   r"   r7   r   ÚrangeÚappendÚlenr6   ÚnpÚlinspaceÚtolistÚsetÚaddÚarray)rP   rR   rS   rL   Úmax_frame_idxrb   Ú
target_fpsÚ
timestampsÚframe_indicesÚcurrent_secondÚinv_fpsÚframe_indexÚstartÚendÚseenÚuniqÚidxrX   r\   r]   s                    @@@r&   Úsample_framesz!GlmgaVideoProcessor.sample_frames�   s¥  øøø€ ð  Ð�w x°¸Ñ=Ô=ÐEÝðXñô ð ð
  Ô0ˆØ$ qÑ(ˆØÔ$ÐO­¨m¸h¼lÑ.JÑ(KÔ(KÈaÑ(Oˆà˜O�S�S°´ˆ
å˜ :Ñ-Ñ.Ô.ˆ	Ý˜	 4¤?Ñ3Ô3ˆ	à ¤Ñ-ÐØJÐJÐJÐJµe¸LÑ6IÔ6IÐJÑJÔJˆ
à˜)Ò#Ð#ØbÐbÐbÐbÐbÕQVÐW`ÑQaÔQaÐbÑbÔbˆMˆMàˆMØˆNØ˜*‘nˆGÝ$ \Ñ2Ô2ð ð �Ø˜kÔ*¨nÒ<Ð<Ø" gÑ-�NØ!×(Ò(¨Ñ5Ô5Ð5Ø%¨°GÑ);Ò;Ð;Ø˜øåˆ}ÑÔ 	Ò)Ð)Ý�=Ñ!Ô! QÒ&Ð&Ø¥ L°1Ñ$4°aÑ 8Ô 8�s��à*¨1Ô-¨}¸RÔ/@�s�ÝœK¨¨s°IÅSÐIÑIÔI×PÒPÑRÔRˆMˆMÝ�ÑÔ )Ò+Ð+ÝœK¨¨<¸!Ñ+;¸YÍcÐRÑRÔR×YÒYÑ[Ô[ˆMå‘U”U˜BˆdˆØ ð 	!ð 	!ˆCØ˜$ˆˆØ—’˜‘”�Ø—’˜CÑ Ô Ð øåˆt‰9Œ9�q‰=ð 	"Ø�KŠK˜˜RœÑ!Ô!Ð!åŒx˜¥CÐ(Ñ(Ô(Ð(r%   Úvideosztorch.TensorÚsizeÚresamplez7PILImageResampling | tvF.InterpolationMode | int | Noner0   r/   Úreturnc           	      ó,  •— |j         r|j        st          d|› d�¦  «        ‚|j        dd…         \  }}t	          |||j        d         |||j         |j        ¬¦  «        \  }	}
t          ¦   «                              |t          |	|
¬¦  «        |¬¦  «        S )	z5Resize dynamically based on input video aspect ratio.zI`size` dict must contain 'shortest_edge' and 'longest_edge' keys but got ú.éþÿÿÿNrK   )r-   r.   r,   r0   r/   r1   r2   )r-   r.   )Úimagerz   r{   )rE   rF   r5   ÚshaperB   rN   Úresizer
   )rP   ry   rz   r{   r0   r/   rL   r-   r.   Úresized_heightÚresized_widthrQ   s              €r&   r‚   zGlmgaVideoProcessor.resizeÄ   s¼   ø€ ð Ô!ð 	r¨Ô):ð 	rÝÐpÐimÐpÐpÐpÑqÔqÐqàœ R S SÔ)‰ˆ�Ý(4ØØØ”| A”ØØ+ØÔ)ØÔ(ð)
ñ )
ô )
Ñ%ˆ˜õ ‰wŒw�~Š~ØÝ °}ÐEÑEÔEØð ñ 
ô 
ð 	
r%   r   r   r   c                 óª  — |j         \  }}}}}	| |z  x}
rB|dd…dd…f                              d|
ddd¦  «        }t          j        ||fd¬¦  «        }||
z  }||z  }||z  |	|z  }}|                     ||||||z  ||||z  ||¦
  «
        }|                     dddddd	d
ddd¦
  «
        }|                     |||z  |z  ||z  |z  |z  ¦  «        }||||fS )ztPatchifies each video into flat layout of shape (`seq_len`, `patch_dim`) so we can concat dynamically shaped pixels.Nr^   rK   ©Údimr   é   é   é   é   r   r(   é   é	   )r�   ÚexpandÚtorchÚcatÚviewÚpermuteÚreshape)rP   ry   r   r   r   Ú
batch_sizer,   Úchannelrƒ   r„   ÚpadÚrepeatsÚgrid_tÚgrid_hÚgrid_wÚpatchesÚflatten_patchess                    r&   ÚpatchifyzGlmgaVideoProcessor.patchifyá   sA  € ð JPÌÑFˆ
�J ¨¸ð �+Ð 3Ñ3Ð3ˆ3ð 	Ø˜Q˜Q˜Q   ˜V”n×+Ò+¨B°°R¸¸RÑ@Ô@ˆGÝ”Y ¨Ð0°aÐ8Ñ8Ô8ˆFØ˜#ÑˆJàÐ2Ñ2ˆØ'¨:Ñ5°}È
Ñ7R�ˆà—+’+ØØØØØ�jÑ ØØØ�jÑ ØØñ
ô 
ˆð —/’/ ! Q¨¨1¨a°°A°q¸!¸QÑ?Ô?ˆØ!Ÿ/š/ØØ�V‰O˜fÑ$ØÐ)Ñ)¨JÑ6¸ÑCñ
ô 
ˆð  ¨°Ð6Ð6r%   gp?Údo_convert_rgbÚ	do_resizeÚ
do_rescaleÚrescale_factorÚdo_normalizeÚ
image_meanÚ	image_stdr   Úreturn_tensorsc           	      ó  — t          |¦  «        \  }}i }|                     ¦   «         D ]C\  }}|r|                      |¦  «        }|r |                      |||||z  |z  |¬¦  «        }|||<   ŒDt	          ||¦  «        }t          |¦  «        \  }}i }i }|                     ¦   «         D ]\  }}t          |d         t          j        ¬¦  «        \  }}|                      |||||	|
¦  «        }|  	                    ||||¬¦  «        \  }}}}|||<   |||ggt          |¦  «        z  ||<   Œ€t	          ||¦  «        }t	          ||¦  «        }t          j        |d¬¦  «        } t          j        |¦  «        }!| |!dœ}"t          |"|¬¦  «        S )N)ry   rz   r{   r0   r/   r   )Úchannel_dim)r   r   r   r†   )rI   rJ   )ÚdataÚtensor_type)r   ÚitemsÚconvert_to_rgbr‚   r   r   r   ÚFIRSTÚrescale_and_normalizer�   re   r�   r�   Útensorr   )#rP   ry   rž   rŸ   rz   r{   r    r¡   r¢   r£   r¤   r   r   r   r   r¥   rL   Úgrouped_videosÚgrouped_videos_indexÚresized_videos_groupedr�   Ústacked_videosÚresized_videosÚprocessed_videos_groupedÚprocessed_gridsrƒ   r„   r›   r˜   r™   rš   Úprocessed_videosrI   rJ   r¨   s#                                      r&   Ú_preprocesszGlmgaVideoProcessor._preprocess	  s  € õ& 0EÀVÑ/LÔ/LÑ,ˆÐ,Ø!#Ðà%3×%9Ò%9Ñ%;Ô%;ð 	;ð 	;Ñ!ˆE�>Øð EØ!%×!4Ò!4°^Ñ!DÔ!D�Øð Ø!%§¢Ø)ØØ%Ø%¨
Ñ2Ð5HÑHØ$7ð "-ñ "ô "�ð -;Ð" 5Ñ)Ð)Ý'Ð(>Ð@TÑUÔUˆõ 0EÀ^Ñ/TÔ/TÑ,ˆÐ,Ø#%Ð ØˆØ%3×%9Ò%9Ñ%;Ô%;ð 	Vð 	VÑ!ˆE�>Ý,:¸>È!Ô;LÕZjÔZpÐ,qÑ,qÔ,qÑ)ˆN˜Mð "×7Ò7Ø 
¨N¸LÈ*ÐV_ñô ˆNð /3¯mªmØØ%Ø%Ø$7ð	 /<ñ /ô /Ñ+ˆG�V˜V Vð /6Ð$ UÑ+Ø'-¨v°vÐ&>Ð%?Å#ÀnÑBUÔBUÑ%UˆO˜EÑ"Ð"å)Ð*BÐDXÑYÔYÐÝ(¨Ð:NÑOÔOˆÝ#œiÐ(8¸aÐ@Ñ@Ô@ÐÝœ oÑ6Ô6ˆà#6Ø,ð
ð 
ˆõ
  °>ÐBÑBÔBÐBr%   )N).r   r   r   r	   ÚBICUBICr{   rz   r   r   r£   r   r¤   rŸ   r    r¢   rž   Údo_sample_framesr   r   Úmax_durationr   r   Úvalid_kwargsr,   rS   Úmodel_input_namesr   r   r   rO   r   r"   Úfloatrx   r
   r‚   Útupler�   Úlistr�   ÚTensorÚboolr!   r   r·   Ú__classcell__)rQ   s   @r&   rD   rD   f   s  ø€ € € € € à!Ô)€HØ&Ð8KÐLÐL€DØ$Ð&9Ð:€NØ!€JØ€IØ€IØ€JØ€LØ€NØÐØ€JØÐØ€LØ€JØ0€LØ€JØ
€Cà.Ð0@ÐAÐØÐØ€Jð# Ð(EÔ!Fð #ð #ð #ð #ð #ð #ð #'ðA)ð A)àðA)ð �5‰[˜4ÑðA)ð A)ð A)ð A)ðF
àð
ð ð
ð Lð	
ð
 ð
ð ð
ð 
ð
ð 
ð 
ð 
ð 
ð 
ð:&7àð&7ð ð&7ð ð	&7ð
 !ð&7ð 
ˆ~˜s CÐ'Ô	(ð&7ð &7ð &7ð &7ðV  $ØØ $ØN`ÔNhØØ )Ø!Ø15Ø04Ø*.Ø!%Ø*.Ø!%Ø26ð!CCð CCà�U”\Ô"ðCCð ðCCð ð	CCð
 ˜‰oðCCð LðCCð ðCCð ðCCð ðCCð ˜D œKÑ'¨$Ñ.ðCCð ˜4 œ;Ñ&¨Ñ-ðCCð ! 4™ZðCCð ˜$‘JðCCð ! 4™ZðCCð ˜$‘JðCCð  ˜jÑ(¨4Ñ/ð!CCð CCð CCð CCð CCð CCð CCð CCr%   rD   )r(   r)   r*   r+   )!r9   Únumpyrf   r�   Útorchvision.transforms.v2r   ÚtvFÚimage_processing_utilsr   Úimage_utilsr   r   r   r	   r
   r   Úprocessing_utilsr   r   Úutilsr   r   Úvideo_processing_utilsr   Úvideo_utilsr   r   r   r   r"   rB   rD   Ú__all__r$   r%   r&   ú<module>rÍ      sì  ðð* €€€à Ð Ð Ð Ø €€€Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7à 2Ð 2Ð 2Ð 2Ð 2Ð 2ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð 5Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4Ø /Ð /Ð /Ð /Ð /Ð /Ð /Ð /Ø 8Ð 8Ð 8Ð 8Ð 8Ð 8Ø OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ OÐ Oðð ð ð ð  L¸ð ñ ô ð ð8 ØØØ0ð!ð !Øð!àð!ð ð!ð ð	!ð
 ð!ð ð!ð ð!ð !ð !ð !ðH ðeCð eCð eCð eCð eCÐ,ñ eCô eCñ „ðeCðP !Ð
!€€€r%   