§
    9²¸j×3  ã                   óê   — d dl mZmZ ddlmZ ddlmZmZ ddlm	Z	m
Z
mZmZ ddlmZmZ ddlmZmZ  e¦   «         rd dlZd dlmc mZ  G d	„ d
e
d¬¦  «        Ze G d„ de¦  «        ¦   «         ZdgZdS )é    )ÚOptionalÚUnioné   )ÚBatchFeature)Ú
ImageInputÚmake_flat_list_of_images)ÚMultiModalDataÚProcessingKwargsÚProcessorMixinÚUnpack)ÚPreTokenizedInputÚ	TextInput)Úauto_docstringÚis_torch_availableNc                   ó,   — e Zd Zddddœdddœddid	œZd
S )ÚColQwen2ProcessorKwargsÚlongestF)ÚpaddingÚreturn_mm_token_type_idsÚreturn_text_replacement_offsetsÚchannels_firstT)Údata_formatÚdo_convert_rgbÚreturn_tensorsÚpt)Útext_kwargsÚimages_kwargsÚcommon_kwargsN)Ú__name__Ú
__module__Ú__qualname__Ú	_defaults© ó    ú€/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/colqwen2/processing_colqwen2.pyr   r   #   sK   € € € € € ð !Ø(-Ø/4ð
ð 
ð ,Ø"ð
ð 
ð +¨DÐ1ðð €I€I€Ir$   r   F)Útotalc                   ól  ‡ — e Zd ZeZ	 	 	 	 	 d"dedz  dedz  fˆ fd„Ze	 	 d#dedz  de	e
z  ee	         z  ee
         z  dee         defˆ fd	„¦   «         Zd#ˆ fd
„	Z	 	 d#dedz  de	e
z  ee	         z  ee
         z  dz  dee         fˆ fd„Zdededefd„Zd$d„Zed„ ¦   «         Zedefd„¦   «         Z ed¬¦  «        	 d$dedz  defd„¦   «         Z ed¬¦  «        de	ee	         z  defd„¦   «         Z	 	 	 d%deded         f         deded         f         deded         ded ef         ddfd!„Zˆ xZS )&ÚColQwen2ProcessorNÚvisual_prompt_prefixÚquery_prefixc                 óî   •— t          ¦   «                              |||¬¦  «         t          |d¦  «        sdn|j        | _        |pd| _        |pd| _        |                     | j        ¦  «        | _        dS )ar  
        visual_prompt_prefix (`str`, *optional*, defaults to `"<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>Describe the image.<|im_end|><|endoftext|>"`):
            A string that gets tokenized and prepended to the image tokens.
        query_prefix (`str`, *optional*, defaults to `"Query: "`):
            A prefix to be used for the query.
        )Úchat_templateÚimage_tokenz<|image_pad|>zf<|im_start|>user
<|vision_start|><|image_pad|><|vision_end|>Describe the image.<|im_end|><|endoftext|>zQuery: N)ÚsuperÚ__init__Úhasattrr-   r)   r*   Úconvert_tokens_to_idsÚimage_token_id)ÚselfÚimage_processorÚ	tokenizerr,   r)   r*   ÚkwargsÚ	__class__s          €r%   r/   zColQwen2Processor.__init__6   s„   ø€ õ 	‰Œ×Ò˜¨)À=ÐÑQÔQÐQÝ29¸)À]Ñ2SÔ2SÐn˜?˜?ÐYbÔYnˆÔØ$8ð %
Øuð 	Ô!ð )Ð5¨IˆÔØ'×=Ò=¸dÔ>NÑOÔOˆÔÐÐr$   ÚimagesÚtextr6   Úreturnc                 óÆ  •‡ ‡‡	— |�|�t          d¦  «        ‚ ‰ j        ‰ j        fd‰ j        j        i|¤Ž}|d                              dd¦  «        Š	‰	du|d         d<   |�‰	€
‰ j        dz  Š	ˆ ˆ	fd„|D ¦   «         } t          ¦   «         j        d||d	œ|¤Ž}|�Æ|d
         dd…df         |d
         dd…df         z  }t          t          j        |d         |                     ¦   «         ¦  «        ¦  «        }t          d„ |D ¦   «         ¦  «        Št          j        ˆfd„|D ¦   «         ¦  «        |d<   ‰	�)|d                              |d         dk    d¦  «        |d<   |S )aô  
        Returns:
            [`BatchFeature`]: A [`BatchFeature`] with the following fields:

            - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`. If `suffix`
              is provided, the `input_ids` will also contain the suffix input ids.
            - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
              `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not
              `None`).
            - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.
            - **labels** -- Labels compatible with training if `suffix` is not None
        Nz5Only one of text or images can be processed at a timeÚtokenizer_init_kwargsr   ÚsuffixÚreturn_token_type_idsé
   c                 ó,   •— g | ]}‰j         › |› ‰› d �‘ŒS )ú
)r*   )Ú.0Úsampler3   r=   s     €€r%   ú
<listcomp>z.ColQwen2Processor.__call__.<locals>.<listcomp>p   s/   ø€ ÐPÐPÐPÀ�tÔ(Ð<¨&Ð<°&Ð<Ð<Ð<ÐPÐPÐPr$   ©r8   r9   Úimage_grid_thwé   é   Úpixel_valuesc              3   ó0   K  — | ]}|j         d          V — ŒdS )r   N)Úshape)rB   Úpixels     r%   ú	<genexpr>z-ColQwen2Processor.__call__.<locals>.<genexpr>x   s(   è è € Ð!KÐ!K°U %¤+¨a¤.Ð!KÐ!KÐ!KÐ!KÐ!KÐ!Kr$   c                 ó^   •— g | ])}t          j        |d d d ‰|j        d          z
  f¦  «        ‘Œ*S )r   )ÚFÚpadrK   )rB   rL   Úmax_num_patchess     €r%   rD   z.ColQwen2Processor.__call__.<locals>.<listcomp>z   s9   ø€ ÐeÐeÐeÈu•”�u˜q ! Q¨¸%¼+Àa¼.Ñ(HÐIÑJÔJÐeÐeÐer$   Ú	input_idsÚtoken_type_idsr   iœÿÿÿÚlabelsr#   )Ú
ValueErrorÚ_merge_kwargsÚvalid_processor_kwargsr5   Úinit_kwargsÚpopÚquery_augmentation_tokenr.   Ú__call__ÚlistÚtorchÚsplitÚtolistÚmaxÚstackÚmasked_fill)r3   r8   r9   r6   Úoutput_kwargsÚmodel_inputsÚoffsetsrI   rQ   r=   r7   s   `       @@€r%   r[   zColQwen2Processor.__call__M   sÌ  øøøø€ ð& Ð Ð 2ÝÐTÑUÔUÐUà*˜Ô*ØÔ'ð
ð 
à"&¤.Ô"<ð
ð ð
ð 
ˆð
 ˜}Ô-×1Ò1°(¸DÑAÔAˆØ@FÈdÐ@Rˆ�mÔ$Ð%<Ñ=àÐàˆ~ØÔ6¸Ñ;�àPÐPÐPÐPÐPÈ4ÐPÑPÔPˆDà'•u‘w”wÔ'ÐR¨v¸DÐRÐRÀMÐRÐRˆàÐà"Ð#3Ô4°Q°Q°Q¸°TÔ:¸\ÐJZÔ=[Ð\]Ð\]Ð\]Ð_`Ð\`Ô=aÑaˆGÝ¥¤¨L¸Ô,HÈ'Ï.Ê.ÑJZÔJZÑ [Ô [Ñ\Ô\ˆLÝ!Ð!KÐ!K¸lÐ!KÑ!KÔ!KÑKÔKˆOÝ+0¬;ØeÐeÐeÐeÐXdÐeÑeÔeñ,ô ,ˆL˜Ñ(ð Ð!à)5°kÔ)B×)NÒ)NØ Ð!1Ô2°aÒ7¸ñ*ô *�˜XÑ&ð Ðr$   c                 ó    •—  t          ¦   «         j        d||dœ|¤Ž^}}}|�'t          |¦  «        }| j        gt	          |¦  «        z  }||d d fS )NrE   r#   )r.   Úprepare_inputs_layoutr   r)   Úlen)r3   r8   r9   r6   Ú_r7   s        €r%   rg   z'ColQwen2Processor.prepare_inputs_layout„   sf   ø€ Ø8�5™7œ7Ô8Ð\ÀÈTÐ\Ð\ÐU[Ð\Ð\Ðˆ��qØÐÝ-¨fÑ5Ô5ˆFØÔ-Ð.µ°V±´Ñ<ˆDØ�t˜T 4Ð'Ð'r$   c                 óx   •— t          ¦   «                              ||¬¦  «         |€|€t          d¦  «        ‚d S d S )NrE   z&Either text or images must be provided)r.   Úvalidate_inputsrU   )r3   r8   r9   r6   r7   s       €r%   rk   z!ColQwen2Processor.validate_inputs‹   sE   ø€ õ 	‰Œ×Ò v°DÐÑ9Ô9Ð9Øˆ<˜F˜NÝÐEÑFÔFÐFð ˆ<˜N˜Nr$   Úimage_inputsÚ	image_idxc                 ó�   — | j         j        dz  }| j        t          |d         |                              ¦   «         ¦  «        |z  z  S )NrH   rF   )r4   Ú
merge_sizer-   ÚintÚprod)r3   rl   rm   r6   Úmerge_lengths        r%   Úreplace_image_tokenz%ColQwen2Processor.replace_image_token•   sE   € ØÔ+Ô6¸Ñ9ˆØÔ¥3 |Ð4DÔ'EÀiÔ'P×'UÒ'UÑ'WÔ'WÑ#XÔ#XÐ\hÑ#hÑiÐir$   c                 ó@  ‡ ‡‡— i }|�Œt           j                             di ¦  «        Š‰                     |¦  «         ‰                     dd¦  «        p‰ j        j        Šˆˆ fd„|D ¦   «         }ˆfd„|D ¦   «         }|                     ||dœ¦  «         t          di |¤ŽS )a¹  
        Computes the number of placeholder tokens needed for multimodal inputs with the given sizes.
        Args:
            image_sizes (`list[list[int]]`, *optional*):
                The input sizes formatted as (height, width) per each image.
        Returns:
            `MultiModalData`: A `MultiModalData` object holding number of tokens per each of the provided
            input modalities, along with other useful data.
        Nr   ro   c                 ó8   •— g | ]} ‰j         j        g |¢‰‘R Ž ‘ŒS r#   )r4   Úget_number_of_image_patches)rB   Ú
image_sizer   r3   s     €€r%   rD   z@ColQwen2Processor._get_num_multimodal_tokens.<locals>.<listcomp>ª   sE   ø€ ð !ð !ð !àð A�Ô$Ô@Ð\À*Ð\ÈmÐ\Ð\Ð\ð!ð !ð !r$   c                 ó    •— g | ]
}|‰d z  z  ‘ŒS )rH   r#   )rB   Únum_patchesro   s     €r%   rD   z@ColQwen2Processor._get_num_multimodal_tokens.<locals>.<listcomp>®   s"   ø€ ÐdÐdÐdÀ; °
¸A±Ñ!=ÐdÐdÐdr$   )Únum_image_tokensÚnum_image_patchesr#   )r   r"   ÚgetÚupdater4   ro   r	   )r3   Úimage_sizesr6   Úvision_datar{   rz   r   ro   s   `     @@r%   Ú_get_num_multimodal_tokensz,ColQwen2Processor._get_num_multimodal_tokens™   sÝ   øøø€ ð ˆØÐ"Ý3Ô=×AÒAÀ/ÐSUÑVÔVˆMØ× Ò  Ñ(Ô(Ð(Ø&×*Ò*¨<¸Ñ>Ô>ÐaÀ$ÔBVÔBaˆJð!ð !ð !ð !ð !à"-ð!ñ !ô !Ðð  eÐdÐdÐdÐRcÐdÑdÔdÐØ×ÒÐ4DÐ[lÐmÐmÑnÔnÐnåÐ,Ð, Ð,Ð,Ð,r$   c                 óT   — | j         j        }| j        j        }d„ |D ¦   «         }||z   S )Nc                 ó   — g | ]}|d v¯|‘Œ	S ))Úpixel_values_videosÚvideo_grid_thwr#   )rB   Únames     r%   rD   z7ColQwen2Processor.model_input_names.<locals>.<listcomp>º   s*   € ð '
ð '
ð '
Ø¸DÐHqÐ<qÐ<qˆDÐ<qÐ<qÐ<qr$   )r5   Úmodel_input_namesr4   )r3   Útokenizer_input_namesÚimage_processor_input_namess      r%   r†   z#ColQwen2Processor.model_input_names³   sF   € à $¤Ô @ÐØ&*Ô&:Ô&LÐ#ð'
ð '
Ø8ð'
ñ '
ô '
Ð#ð %Ð'BÑBÐBr$   c                 ó   — | j         j        S )zŠ
        Return the query augmentation token.

        Query augmentation buffers are used as reasoning buffers during inference.
        )r5   Ú	pad_token)r3   s    r%   rZ   z*ColQwen2Processor.query_augmentation_token¿   s   € ð Œ~Ô'Ð'r$   zpThis method forwards the `images` and `kwargs` arguments to ColQwen2Processor's [`~ColQwen2Processor.__call__`].)Úcustom_introc                 ó    —  | j         dd|i|¤ŽS )Nr8   r#   ©r[   )r3   r8   r6   s      r%   Úprocess_imagesz ColQwen2Processor.process_imagesÈ   s!   € ð ˆtŒ}Ð5Ð5 FÐ5¨fÐ5Ð5Ð5r$   znThis method forwards the `text` and `kwargs` arguments to ColQwen2Processor's [`~ColQwen2Processor.__call__`].c                 ó    —  | j         dd|i|¤ŽS )Nr9   r#   r�   )r3   r9   r6   s      r%   Úprocess_queriesz!ColQwen2Processor.process_queriesÒ   s!   € ð ˆtŒ}Ð1Ð1 $Ð1¨&Ð1Ð1Ð1r$   é€   ÚcpuÚquery_embeddingsztorch.TensorÚpassage_embeddingsÚ
batch_sizeÚoutput_dtypeztorch.dtypeÚoutput_deviceztorch.devicec           	      ó8  — t          |¦  «        dk    rt          d¦  «        ‚t          |¦  «        dk    rt          d¦  «        ‚|d         j        |d         j        k    rt          d¦  «        ‚|d         j        |d         j        k    rt          d¦  «        ‚|€|d         j        }g }t	          dt          |¦  «        |¦  «        D �]:}g }t
          j        j        j         	                    ||||z   …         dd¬¦  «        }	t	          dt          |¦  «        |¦  «        D ]�}
t
          j        j        j         	                    ||
|
|z   …         dd¬¦  «        }| 
                    t          j        d	|	|¦  «                             d
¬¦  «        d                              d¬¦  «        ¦  «         Œ‘| 
                    t          j        |d¬¦  «                             |¦  «                             |¦  «        ¦  «         �Œ<t          j        |d¬¦  «        S )a[  
        Compute the late-interaction/MaxSim score (ColBERT-like) for the given multi-vector
        query embeddings (`qs`) and passage embeddings (`ps`). For ColQwen2, a passage is the
        image of a document page.

        Because the embedding tensors are multi-vector and can thus have different shapes, they
        should be fed as:
        (1) a list of tensors, where the i-th tensor is of shape (sequence_length_i, embedding_dim)
        (2) a single tensor of shape (n_passages, max_sequence_length, embedding_dim) -> usually
            obtained by padding the list of tensors.

        Args:
            query_embeddings (`Union[torch.Tensor, list[torch.Tensor]`): Query embeddings.
            passage_embeddings (`Union[torch.Tensor, list[torch.Tensor]`): Passage embeddings.
            batch_size (`int`, *optional*, defaults to 128): Batch size for computing scores.
            output_dtype (`torch.dtype`, *optional*, defaults to `torch.float32`): The dtype of the output tensor.
                If `None`, the dtype of the input embeddings is used.
            output_device (`torch.device` or `str`, *optional*, defaults to "cpu"): The device of the output tensor.

        Returns:
            `torch.Tensor`: A tensor of shape `(n_queries, n_passages)` containing the scores. The score
            tensor is saved on the "cpu" device.
        r   zNo queries providedzNo passages providedz/Queries and passages must be on the same devicez-Queries and passages must have the same dtypeNT)Úbatch_firstÚpadding_valuezbnd,csd->bcnsr   )ÚdimrH   rG   )rh   rU   ÚdeviceÚdtypeÚranger]   ÚnnÚutilsÚrnnÚpad_sequenceÚappendÚeinsumr`   ÚsumÚcatÚto)r3   r“   r”   r•   r–   r—   ÚscoresÚiÚbatch_scoresÚbatch_queriesÚjÚbatch_passagess               r%   Úscore_retrievalz!ColQwen2Processor.score_retrievalÜ   s*  € õ@ ÐÑ Ô  AÒ%Ð%ÝÐ2Ñ3Ô3Ð3ÝÐ!Ñ"Ô" aÒ'Ð'ÝÐ3Ñ4Ô4Ð4à˜AÔÔ%Ð);¸AÔ)>Ô)EÒEÐEÝÐNÑOÔOÐOà˜AÔÔ$Ð(:¸1Ô(=Ô(CÒCÐCÝÐLÑMÔMÐMàÐØ+¨AÔ.Ô4ˆLà%'ˆå�q�#Ð.Ñ/Ô/°Ñ<Ô<ð 	]ñ 	]ˆAØ/1ˆLÝ!œHœNÔ.×;Ò;Ø   Q¨¡^Ð!3Ô4À$ÐVWð <ñ ô ˆMõ ˜1�cÐ"4Ñ5Ô5°zÑBÔBð ð �Ý!&¤¤Ô!3×!@Ò!@Ø& q¨1¨z©>Ð'9Ô:ÈÐ\]ð "Añ "ô "�ð ×#Ò#Ý”L °-ÀÑPÔP×TÒTÐYZÐTÑ[Ô[Ð\]Ô^×bÒbÐghÐbÑiÔiñô ð ð ð �MŠM�%œ) L°aÐ8Ñ8Ô8×;Ò;¸LÑIÔI×LÒLÈ]Ñ[Ô[Ñ\Ô\Ð\Ñ\åŒy˜ QÐ'Ñ'Ô'Ð'r$   )NNNNN)NN)N)r‘   Nr’   )r   r    r!   r   rW   Ústrr/   r   r   r   r   r\   r   r   r[   rg   r
   rk   Údictrp   rs   r€   Úpropertyr†   rZ   rŽ   r�   r   r   r®   Ú__classcell__)r7   s   @r%   r(   r(   2   s`  ø€ € € € € à4Ðð ØØØ+/Ø#'ðPð Pð
 " D™jðPð ˜D‘jðPð Pð Pð Pð Pð Pð. ð %)ØZ^ð4ð 4à˜TÑ!ð4ð Ð+Ñ+¨d°9¬oÑ=ÀÐEVÔ@WÑWð4ð Ð0Ô1ð	4ð
 
ð4ð 4ð 4ð 4ð 4ñ „^ð4ðl(ð (ð (ð (ð (ð (ð %)ØaeðGð Gà˜TÑ!ðGð Ð+Ñ+¨d°9¬oÑ=ÀÐEVÔ@WÑWÐZ^Ñ^ðGð Ð)Ô*ð	Gð Gð Gð Gð Gð Gðj°ð jÀð jÐSVð jð jð jð jð-ð -ð -ð -ð4 ð	Cð 	Cñ „Xð	Cð ð(¨#ð (ð (ð (ñ „Xð(ð €^ð Hðñ ô ð
 %)ð6ð 6à˜TÑ!ð6ð 
ð	6ð 6ð 6ñô ð6ð €^ð Fðñ ô ð2à˜$˜yœ/Ñ)ð2ð 
ð	2ð 2ð 2ñô ð2ð Ø04Ø49ð>(ð >(à °°^Ô0DÐ DÔEð>(ð " .°$°~Ô2FÐ"FÔGð>(ð ð	>(ð
 ˜}Ô-ð>(ð ˜^¨SÐ0Ô1ð>(ð 
ð>(ð >(ð >(ð >(ð >(ð >(ð >(ð >(r$   r(   )Útypingr   r   Úfeature_extraction_utilsr   Úimage_utilsr   r   Úprocessing_utilsr	   r
   r   r   Útokenization_utils_baser   r   r    r   r   r]   Útorch.nn.functionalrŸ   Ú
functionalrO   r   r(   Ú__all__r#   r$   r%   ú<module>r»      sS  ðð* #Ð "Ð "Ð "Ð "Ð "Ð "Ð "à 4Ð 4Ð 4Ð 4Ð 4Ð 4Ø ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ø XÐ XÐ XÐ XÐ XÐ XÐ XÐ XÐ XÐ XÐ XÐ XØ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CØ 7Ð 7Ð 7Ð 7Ð 7Ð 7Ð 7Ð 7ð ÐÑÔð $Ø€L€L€LØ#Ð#Ð#Ð#Ð#Ð#Ð#Ð#Ð#ðð ð ð ð Ð.°eð ñ ô ð ð ðg(ð g(ð g(ð g(ð g(˜ñ g(ô g(ñ „ðg(ðT Ð
€€€r$   