§
    9²¸jxq  ã                   óœ  — d Z ddlZddlZddlmZ ddlZddlmZ ddl	m
Z
mZ ddlmZmZmZmZ ddlmZmZ dd	lmZmZmZmZ dd
lmZ  ej        e¦  «        Z e¦   «         rddlZdZdZ dZ!dZ"dZ#dZ$dZ%dZ&dZ' G d„ ded¬¦  «        Z(d„ Z)d„ Z*de+de+de+de,e-         fd„Z.de+d e+d!e+d"e+de+de,e-         fd#„Z/d$e0de0fd%„Z1d$e0de,fd&„Z2d'e0de+de0fd(„Z3d'e,e0         d)e,d*         de,e0         fd+„Z4d,ee,e,e-                  d*f         d-d*d.e,e,d*                  d/e-de,d*         f
d0„Z5 ed1¬2¦  «        e G d3„ d4e¦  «        ¦   «         ¦   «         Z6d4gZ7dS )5z$
Image/Text processor class for GIT
é    N)ÚUnioné   )ÚBatchFeature)Ú
ImageInputÚmake_flat_list_of_images)ÚMultiModalDataÚProcessingKwargsÚProcessorMixinÚUnpack)ÚPreTokenizedInputÚ	TextInput)Úauto_docstringÚis_torch_availableÚloggingÚrequires_backends)Úrequiresz<box>z</box>z<point>z</point>z<0x00>z<0x01>z<0x02>z<0x03>z<0x04>c                   ó   — e Zd ZddddœiZdS )ÚFuyuProcessorKwargsÚtext_kwargsTF)Úadd_special_tokensÚpaddingN)Ú__name__Ú
__module__Ú__qualname__Ú	_defaults© ó    úx/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/models/fuyu/processing_fuyu.pyr   r   8   s(   € € € € € àØ"&Øð
ð 
ð€I€I€Ir   r   F)Útotalc                 ój   — t          j        | |z  ¦  «                             t           j        ¦  «        S ©N©ÚnpÚroundÚastypeÚint32)Úoriginal_coordsÚscale_hs     r   Ú original_to_transformed_h_coordsr)   B   ó'   € ÝŒ8�O gÑ-Ñ.Ô.×5Ò5µb´hÑ?Ô?Ð?r   c                 ój   — t          j        | |z  ¦  «                             t           j        ¦  «        S r!   r"   )r'   Úscale_ws     r   Ú original_to_transformed_w_coordsr-   G   r*   r   ÚxÚyÚscale_factorÚreturnc                 óº   — t          t          j        | dz  g¦  «        |¦  «        d         }t          t          j        |dz  g¦  «        |¦  «        d         }||gS ©Né   r   ©r-   r#   Úarrayr)   )r.   r/   r0   Úx_scaledÚy_scaleds        r   Ú scale_point_to_transformed_imager9   K   sW   € Ý/µ´¸!¸a¹%¸Ñ0AÔ0AÀ<ÑPÔPÐQRÔS€HÝ/µ´¸!¸a¹%¸Ñ0AÔ0AÀ<ÑPÔPÐQRÔS€HØ�hÐÐr   ÚtopÚleftÚbottomÚrightc                 ón  — t          t          j        | dz  g¦  «        |¦  «        d         }t          t          j        |dz  g¦  «        |¦  «        d         }t          t          j        |dz  g¦  «        |¦  «        d         }t          t          j        |dz  g¦  «        |¦  «        d         }||||gS r3   r5   )	r:   r;   r<   r=   r0   Ú
top_scaledÚleft_scaledÚbottom_scaledÚright_scaleds	            r   Úscale_bbox_to_transformed_imagerC   Q   s§   € õ 2µ"´(¸CÀ!¹G¸9Ñ2EÔ2EÀ|ÑTÔTÐUVÔW€JÝ2µ2´8¸TÀA¹X¸JÑ3GÔ3GÈÑVÔVÐWXÔY€KÝ4µR´X¸vÈ¹z¸lÑ5KÔ5KÈ\ÑZÔZÐ[\Ô]€MÝ3µB´H¸eÀa¹i¸[Ñ4IÔ4IÈ<ÑXÔXÐYZÔ[€LØ˜ ]°LÐAÐAr   Úpromptc                 ó  — |                       t          t          ¦  «        } |                       t          t          ¦  «        } |                       t
          t          ¦  «        } |                       t          t          ¦  «        } | S r!   )	ÚreplaceÚTEXT_REPR_POINT_OPENÚTOKEN_POINT_OPEN_STRINGÚTEXT_REPR_POINT_CLOSEÚTOKEN_POINT_CLOSE_STRINGÚTEXT_REPR_BBOX_OPENÚTOKEN_BBOX_OPEN_STRINGÚTEXT_REPR_BBOX_CLOSEÚTOKEN_BBOX_CLOSE_STRING)rD   s    r   Ú$_replace_string_repr_with_token_tagsrO   [   s[   € Ø�^Š^Õ0Õ2IÑJÔJ€FØ�^Š^Õ1Õ3KÑLÔL€FØ�^Š^Õ/Õ1GÑHÔH€FØ�^Š^Õ0Õ2IÑJÔJ€FØ€Mr   c                 ó�  — g }t          j        dt          › dt          › dt          › dt
          › d�	¦  «        }|                     | ¦  «        }t          |¦  «        D ]i\  }}t          |¦  «        dk    s|t          t          t          t
          fv rŒ5| 	                    ||dk    o||dz
           t          t          fv f¦  «         Œj|S )zY
    Given a string prompt, converts the prompt into a list of TextTokenConversions.
    ú(ú|ú)r   é   )
ÚreÚcompilerL   rN   rH   rJ   ÚsplitÚ	enumerateÚlenÚappend)rD   Úprompt_text_listÚregex_patternÚprompt_splitÚiÚelems         r   Ú+_segment_prompt_into_text_token_conversionsr`   c   sñ   € ð
  ÐÝ”JØtÕ"ÐtÐtÕ%<ÐtÐtÕ?VÐtÐtÕYqÐtÐtÐtñô €Mð !×&Ò& vÑ.Ô.€LÝ˜\Ñ*Ô*ð 

ð 

‰ˆˆ4Ýˆt‰9Œ9˜Š>ˆ>˜TÝ"Ý#Ý#Ý$ð	&
ð 
ð 
ð Ø×ÒØ�1�q’5Ðe˜\¨!¨a©%Ô0Õ5KÕMdÐ4eÐeÐfñ	
ô 	
ð 	
ð 	
ð Ðr   Útextc                 ó  — |                       d¦  «        }t          |¦  «        dk    rt          }t          }nt          }t
          }d„ |D ¦   «         }t          |¦  «        dk    rt          |d         |d         |¬¦  «        }n_t          |¦  «        dk    r-t          |d         |d         |d         |d         |¬	¦  «        }nt          d
t          |¦  «        › �¦  «        ‚d 	                    d„ |D ¦   «         ¦  «        }||z   |z   S )z¿
    Given a bounding box of the fashion <box>1, 2, 3, 4</box> | <point>1, 2</point> This function is responsible for
    converting 1, 2, 3, 4 into tokens of 1 2 3 4 without any commas.
    ú,r4   c                 óP   — g | ]#}t          |                     ¦   «         ¦  «        ‘Œ$S r   )ÚfloatÚstrip©Ú.0Únums     r   ú
<listcomp>z)transform_within_tags.<locals>.<listcomp>Œ   s(   € Ð;Ð;Ð; s•�c—i’i‘k”kÑ"Ô"Ð;Ð;Ð;r   r   rT   )r.   r/   r0   é   r   )r:   r;   r<   r=   r0   zInvalid number of ints: Ú c                 ó,   — g | ]}t          |¦  «        ‘ŒS r   )Ústrrg   s     r   rj   z)transform_within_tags.<locals>.<listcomp>š   s   € Ð>Ð>Ð> 3•c˜#‘h”hÐ>Ð>Ð>r   )
rW   rY   rH   rJ   rL   rN   r9   rC   Ú
ValueErrorÚjoin)ra   r0   Únum_int_strsÚtoken_space_open_stringÚtoken_space_close_stringÚnum_intsÚnum_ints_translatedÚtokenss           r   Útransform_within_tagsrw   |   s$  € ð —:’:˜c‘?”?€LÝ
ˆ<ÑÔ˜AÒÐå"9ÐÝ#;Ð Ð å"8ÐÝ#:Ð ð <Ð;¨lÐ;Ñ;Ô;€Hå
ˆ8�}„}˜ÒÐÝ>ÀÈ!ÄÐPXÐYZÔP[ÐjvÐwÑwÔwÐÐÝ	ˆX‰Œ˜!Ò	Ð	Ý=Ø˜”Ø˜!”Ø˜A”;Ø˜1”+Ø%ð
ñ 
ô 
ÐÐõ ÐCµC¸±M´MÐCÐCÑDÔDÐDØ�WŠWÐ>Ð>Ð*=Ð>Ñ>Ô>Ñ?Ô?€FØ" VÑ+Ð.FÑFÐFr   Úscale_factorsztorch.Tensorc                 ó’  — g }t          | |¦  «        D ]³\  }}t          |¦  «        }t          |¦  «        }g }|D ]c}|d         r>t          |d         |                     ¦   «         ¦  «        }|                     |¦  «         ŒH|                     |d         ¦  «         Œd|                     d                     |¦  «        ¦  «         Œ´|S )añ  
    This function transforms the prompts in the following fashion:
    - <box> <point> and </box> </point> to their respective token mappings
    - extract the coordinates from the tag
    - transform the coordinates into the transformed image space
    - return the prompts with the transformed coordinates and new tags

    Bounding boxes and points MUST be in the following format: <box>y1, x1, y2, x2</box> <point>x, y</point> The spaces
    and punctuation added above are NOT optional.
    rT   r   rl   )ÚziprO   r`   rw   ÚitemrZ   rp   )	ra   rx   Úprocessed_textrD   r0   r[   Útransformed_promptsr_   Úprompt_within_tagss	            r   Útransform_coordinatesr   ž   sà   € ð €NÝ # D¨-Ñ 8Ô 8ð <ð <Ñˆ�Ý5°fÑ=Ô=ˆÝFÀvÑNÔNÐØ)+ÐØ$ð 	4ð 	4ˆDØ�AŒwð 4å%:¸4À¼7ÀL×DUÒDUÑDWÔDWÑ%XÔ%XÐ"Ø#×*Ò*Ð+=Ñ>Ô>Ð>Ð>à#×*Ò*¨4°¬7Ñ3Ô3Ð3Ð3Ø×Ò˜bŸgšgÐ&9Ñ:Ô:Ñ;Ô;Ð;Ð;ØÐr   Únum_real_text_tokensÚinput_streamÚimage_tokensÚ
batch_sizec                 óê   — t          j        |d         d         |d         gd¬¦  «        }|d         d         j        d         | d         d         z   }t          j        |d|…         gd¬¦  «        g}|S )a  Takes an input_stream tensor of shape B x S x ?. For each subsequence, adds any required
    padding to account for images and then unpacks the subsequences to create a single sequence per item in the batch.
    Returns a list of tensors, one for each item in the batch.r   )r   r   ©ÚdimN)ÚtorchÚcatÚshape)r€   r�   r‚   rƒ   Úsubsequence_streamÚnum_real_tokensÚall_bi_streams          r   Úconstruct_full_unpacked_streamr�   ¹   s   € õ œ L°¤O°AÔ$6¸ÀTÔ8JÐ#KÐQRÐSÑSÔSÐØ" 1”o aÔ(Ô.¨qÔ1Ð4HÈÔ4KÈAÔ4NÑN€OÝ”YÐ 2Ð3C°OÐ3CÔ DÐEÈ1ÐMÑMÔMÐN€MàÐr   )Úvision)Úbackendsc            
       óð  ‡ — e Zd ZeZe	 dd„¦   «         Zˆ fd„Zede	e
         fd„¦   «         Ze	 	 ddedz  dee	e         z  ez  ez  dz  d	ee         dd
fd„¦   «         Z	 	 ddedz  deez  e	e         z  e	e         z  fd„Z	 	 ddee	e         z  dz  deez  e	e         z  e	e         z  d	ee         fˆ fd„Zdefd„Zdede
defd„Zede	e         fd„¦   «         Zdd„Zdd„Zdd„Zed„ ¦   «         Zˆ xZS )ÚFuyuProcessorrl   c                 ót   — ddl m}  |j        |fi |¤Ž}d|j        v r|j                             d¦  «         |S )zÝ
        Override for BC. Fuyu uses TokenizersBackend and requires token_type_ids to be removed from model_input_names
        because Fuyu uses mm_token_type_ids instead for multimodal token identification.    `
        r   )ÚTokenizersBackendÚtoken_type_ids)Útokenization_utils_tokenizersr“   Úfrom_pretrainedÚmodel_input_namesÚremove)ÚclsÚsub_processor_typeÚpretrained_model_name_or_pathÚ	subfolderÚkwargsr“   Ú	tokenizers          r   Ú_load_tokenizer_from_pretrainedz-FuyuProcessor._load_tokenizer_from_pretrainedÏ   s`   € ð 	GÐFÐFÐFÐFÐFà5Ð%Ô5Ð6SÐ^Ð^ÐW]Ð^Ð^ˆ	à˜yÔ:Ð:Ð:ØÔ'×.Ò.Ð/?Ñ@Ô@Ð@ØÐr   c                 óâ   •— d| _         |                     ¦   «         }d|_        d| _        d| _        |d         | _        |d         | _        t          ¦   «                              ||¬¦  «         d S )Né
   r   z	|SPEAKER|z	|NEWLINE|)Úimage_processorrž   )	Úmax_tokens_to_generateÚ	get_vocabÚpad_token_idÚimage_tokenÚimage_newline_tokenÚimage_token_idÚimage_newline_idÚsuperÚ__init__)Úselfr¢   rž   r�   ÚvocabÚ	__class__s        €r   r«   zFuyuProcessor.__init__ß   sq   ø€ Ø&(ˆÔ#Ø×#Ò#Ñ%Ô%ˆØ!"ˆ	Ôà&ˆÔØ#.ˆÔ Ø# KÔ0ˆÔØ % kÔ 2ˆÔÝ‰Œ×Ò¨ÀIÐÑNÔNÐNÐNÐNr   r1   c                 ó   — | j         | j        gS r!   )r©   r¨   ©r¬   s    r   Úimage_token_idszFuyuProcessor.image_token_idsê   s   € àÔ% tÔ':Ð;Ð;r   NÚimagesra   r�   r   c           	      óT  ‡ — t          ‰ dg¦  «          ‰ j        t          fd‰ j        j        i|¤Ž}|d                              dd¦  «        st          d¦  «        ‚|�|�d|d         d	<   d|d         d
<   |�d|d         d<   d|d         d<    ‰ j        d'||dœ|¤Ž^}}} ‰ j        d'||dœ|¤Ž i }g }|�'t          ‰ d¦  «        r ‰ j
        |fi |d         ¤Ž\  }}i }|d                              dd¦  «        }	|��m|d                              dd¦  «        }
|d                              dd¦  «        }|                     d¦  «        x}�d„ t          ||¦  «        D ¦   «         }|r‰                      ||¦  «        \  }}|s ‰ j        |fi |d         ¤Ž}�n�g g }}|D �]g}t          j        d|d¬¦  «        } ‰ j        |d         fi |d         ¤Ž}t#          |¦  «        dk    rÄ ‰ j        |d         fi |d         ¤Ž}|                     t'          j        |d         ddd…f         |d         gd¬¦  «                             d¦  «        ¦  «         |                     t'          j        |d         ddd…f         |d         gd¬¦  «                             d¦  «        ¦  «         �Œ|                     |d                              d¦  «        ¦  «         |                     |d                              d¦  «        ¦  «         �Œi‰ j                             ||dœdd¬ ¦  «        }|r‰                      ||d!g¬"¦  «         |r||d#<   |
r‰                      |d         ¦  «        |d$<   i |¥|¥}ˆ fd%„|                     ¦   «         D ¦   «         }t5          ||	‰ j        ¬&¦  «        S )(aè  
        Returns:
            [`FuyuBatchEncoding`]: A [`FuyuBatchEncoding`] with the following fields:

            - **input_ids** -- Tensor of token ids to be fed to a model. Returned when `text` is not `None`.
            - **image_patches** -- List of Tensor of image patches. Returned when `images` is not `None`.
            - **attention_mask** -- List of indices specifying which tokens should be attended to by the model when
              `return_attention_mask=True`.
        r‡   Útokenizer_init_kwargsr   Úreturn_attention_maskTz>`return_attention_mask=False` is not supported for this model.NFr   r   ÚptÚreturn_tensorsÚimages_kwargs©r²   ra   r¢   Úreturn_mm_token_type_idsÚreturn_text_replacement_offsetsÚimage_scale_factorsc                 ó4   — g | ]\  }}t          ||¦  «        ‘ŒS r   )r   )rh   ÚsampleÚscales      r   rj   z*FuyuProcessor.__call__.<locals>.<listcomp>&  s'   € ÐsÐsÐsÁ=À6È5Õ/°¸Ñ>Ô>ÐsÐsÐsr   z(?<=<s>)rT   )Úmaxsplitéÿÿÿÿr4   r   Ú	input_ids.r…   Úattention_mask)rÂ   rÃ   r;   )Úpadding_sideÚ
max_lengthÚimage)Ú
modalitiesÚtext_replacement_offsetsÚmm_token_type_idsc                 ó.   •— i | ]\  }}|‰j         v¯||“ŒS r   )Úunused_input_names)rh   ÚkÚvr¬   s      €r   ú
<dictcomp>z*FuyuProcessor.__call__.<locals>.<dictcomp>]  s,   ø€ ÐRÐRÐR™˜˜A°¸$Ô:QÐ1QÐ1Q��1Ð1QÐ1QÐ1Qr   )Útensor_typeÚskip_tensor_conversionr   )r   Ú_merge_kwargsr   rž   Úinit_kwargsÚ
setdefaultro   Úprepare_inputs_layoutÚvalidate_inputsÚhasattrÚ_process_imagesÚgetÚpoprz   Úget_text_with_replacementsrU   rW   rY   rZ   r‡   rˆ   ÚsqueezeÚpadÚ_check_special_mm_tokensÚcreate_mm_token_type_idsÚitemsr   rÐ   )r¬   r²   ra   r�   Úmerged_kwargsÚ_Úprocessed_imagesÚimages_replacementsÚtext_inputsr·   rº   r»   r¼   r¾   rÈ   Úbatch_input_idsÚbatch_attention_maskÚsplit_sampleÚprompt_inputsÚplaceholder_inputsÚdatas   `                    r   Ú__call__zFuyuProcessor.__call__î   sÙ  ø€ õ  	˜$  	Ñ*Ô*Ð*à*˜Ô*Ýð
ð 
à"&¤.Ô"<ð
ð ð
ð 
ˆð
 ˜]Ô+×6Ò6Ð7NÐPTÑUÔUð 	_ÝÐ]Ñ^Ô^Ð^ð Ð Ð 2ØAFˆM˜-Ô(Ð)=Ñ>Ø6;ˆM˜-Ô(¨Ñ3àÐØ=AˆM˜-Ô(Ð)9Ñ:Ø?CˆM˜/Ô*Ð+;Ñ<ð 6˜4Ô5ÐY¸VÈ$ÐYÐYÐRXÐYÐYÐˆ��qØˆÔÐ@ F°Ð@Ð@¸Ð@Ð@Ð@àÐØ ÐØÐ¥'¨$Ð0AÑ"BÔ"BÐØ4H°DÔ4HÈÐ4rÐ4rÐS`ÐapÔSqÐ4rÐ4rÑ1ÐÐ1àˆØ& }Ô5×9Ò9Ð:JÈDÑQÔQˆØÑØ'4°]Ô'C×'GÒ'GÐHbÐdiÑ'jÔ'jÐ$Ø.;¸MÔ.J×.NÒ.NØ1°5ñ/ô /Ð+ð (8×';Ò';Ð<QÑ'RÔ'RÐRÐ#Ð_ØsÐsÕTWÐX\Ð^qÑTrÔTrÐsÑsÔs�à"ð Ø15×1PÒ1PØØ'ñ2ô 2Ñ.�Ð.ð 'ð !Ø,˜dœn¨TÐRÐR°]À=Ô5QÐRÐR�‘ð 9;¸BÐ!5�Ø"ð `ñ `�FÝ#%¤8¨K¸È!Ð#LÑ#LÔ#L�LØ$2 D¤N°<ÀÔ3CÐ$dÐ$dÀ}ÐUbÔGcÐ$dÐ$d�Må˜<Ñ(Ô(¨AÒ-Ð-ð .<¨T¬^¸LÈ¼OÐ-lÐ-lÈ}Ð]jÔOkÐ-lÐ-lÐ*Ø'×.Ò.Ý!œIØ!3°KÔ!@ÀÀaÀbÀbÀÔ!IÈ=ÐYdÔKeÐ fÐlnðñ ô ç%šg a™jœjñô ð ð
 -×3Ò3Ý!œIØ!3Ð4DÔ!EÀcÈ1È2È2ÀgÔ!NÐP]Ð^nÔPoÐ pØ$&ðñ ô ÷ &šg a™jœjñ	ô ð ñ ð (×.Ò.¨}¸[Ô/I×/QÒ/QÐRSÑ/TÔ/TÑUÔUÐUØ,×3Ò3°MÐBRÔ4S×4[Ò4[Ð\]Ñ4^Ô4^Ñ_Ô_Ð_Ñ_ð #œn×0Ò0Ø"1ÐEYÐZÐZØ!'Ø#ð 1ñ ô �ð #ð WØ×-Ò-¨d°KÈWÈIÐ-ÑVÔVÐVà.ð SØ:R�Ð6Ñ7à'ð kØ37×3PÒ3PÐQ\Ð]hÔQiÑ3jÔ3j�Ð/Ñ0ð 3�+Ð2Ð!1Ð2ˆØRÐRÐRÐR §¢¡¤ÐRÑRÔRˆÝ˜D¨nÐUYÔUpÐqÑqÔqÐqr   c                 ó  — |�˜|�–g }t          |t          ¦  «        r|gn|}t          |t          t          f¦  «        s|gn|}t	          ||¦  «        D ]G\  }}t          |t          ¦  «        r|g k    s
| j        |z   }|                     |t          z   ¦  «         ŒHn@|�t           	                    d¦  «         |}n!|€|�t           	                    d¦  «         dg}|�)| j
                             |¦  «        }t          |¦  «        }||d d fS )NzMYou are processing a text with no associated image. Make sure it is intended.zNYou are processing an image with no associated text. Make sure it is intended.rl   )Ú
isinstancern   ÚlistÚtuplerz   r¦   rZ   ÚBEGINNING_OF_ANSWER_STRINGÚloggerÚwarningr¢   Úfetch_imagesr   )r¬   r²   ra   r�   Úprepared_textr¾   rÆ   s          r   rÔ   z#FuyuProcessor.prepare_inputs_layout`  s5  € ð Ð Ð 2ØˆMÝ'¨­cÑ2Ô2Ð<�D�6�6¸ˆDÝ%/°½½u¸Ñ%FÔ%FÐR�f�X�XÈFˆFÝ!$ T¨6Ñ!2Ô!2ð Jð J‘�˜Ý" 5­$Ñ/Ô/ð 7°E¸R²K°KØ!Ô-°Ñ6�Fð ×$Ò$ VÕ.HÑ%HÑIÔIÐIÐIðJð ÐÝ�NŠNÐjÑkÔkÐkØ ˆMˆMØˆ\˜fÐ0Ý�NŠNÐkÑlÔlÐlØ˜DˆMàÐØÔ)×6Ò6°vÑ>Ô>ˆFÝ-¨fÑ5Ô5ˆFà�} d¨DÐ0Ð0r   c                 ój   •—  t          ¦   «         j        d||dœ|¤Ž |€|€t          d¦  «        ‚d S d S )Nr¹   z+You must provide either `text` or `images`.r   )rª   rÕ   ro   )r¬   r²   ra   r�   r®   s       €r   rÕ   zFuyuProcessor.validate_inputs€  sL   ø€ ð 	 �‰ŒÔÐC v°DÐCÐC¸FÐCÐCÐCàˆ<˜F˜NÝÐJÑKÔKÐKð ˆ<˜N˜Nr   c                 ó\  —  | j         |fi |¤Ž}g }g }|                     d¦  «        }t          t          |¦  «        ¦  «        D �]K}||         j        dd …         \  }}	| j         j        }
|
j        |
j        }}t          |t          j
        |d         |         d         |z  ¦  «        |z  ¦  «        }t          |	t          j
        |d         |         d         |z  ¦  «        |z  ¦  «        }| j                              |||
¬¦  «        }||         dd |…d |…f         }| j                              ||
¬¦  «        }|                     |                     d¦  «        ¦  «         ||z  } | j        |f|||d	œ|¤Ž}|                     |¦  «         �ŒMt!          j        |d¬
¦  «        |d<   ||fS )Nr²   éþÿÿÿÚimage_unpadded_heightsr   Úimage_unpadded_widths)Úimage_heightÚimage_widthÚ
patch_size.)rÆ   rü   )Ú	image_idxÚnum_patchesÚ	row_widthr…   Úimage_patches)r¢   rÙ   ÚrangerY   r‰   rü   ÚheightÚwidthÚminÚmathÚceilÚget_num_patchesÚpatchify_imagerZ   rÛ   Úreplace_image_tokenr‡   rˆ   )r¬   r²   r�   râ   Úbatch_image_patchesÚimage_replacementsÚprocessor_output_imagesrý   rú   rû   rü   Úpatch_heightÚpatch_widthÚnew_hÚnew_wrþ   rÆ   r   rÿ   Úreplacement_texts                       r   r×   zFuyuProcessor._process_images‹  s  € Ø/˜4Ô/°ÐAÐA¸&ÐAÐAÐà ÐØÐØ"2×"6Ò"6°xÑ"@Ô"@ÐÝ�s 6™{œ{Ñ+Ô+ð 	8ñ 	8ˆIØ(?À	Ô(JÔ(PÐQSÐQTÐQTÔ(UÑ%ˆL˜+ØÔ-Ô8ˆJØ(2Ô(9¸:Ô;K˜+ˆLåØÝ”	Ð*Ð+CÔDÀYÔOÐPQÔRÐUaÑaÑbÔbÐeqÑqñô ˆEõ ØÝ”	Ð*Ð+BÔCÀIÔNÈqÔQÐT_Ñ_Ñ`Ô`ÐcnÑnñô ˆEð Ô.×>Ò>Ø"°À*ð ?ñ ô ˆKð ,¨IÔ6°s¸F¸U¸FÀFÀUÀFÐ7JÔKˆEØ Ô0×?Ò?ÀeÐXbÐ?ÑcÔcˆMØ×&Ò& }×'<Ò'<¸QÑ'?Ô'?Ñ@Ô@Ð@à Ñ,ˆIØ7˜tÔ7Ø ð Ø,5À;ÐZcð ð  Øgmð ð  Ðð ×%Ò%Ð&6Ñ7Ô7Ð7Ñ7å,1¬IÐ6IÈqÐ,QÑ,QÔ,QÐ˜Ñ)ØÐ!3Ð3Ð3r   Úimage_inputsrý   c                 óÌ   — g }t          d|d         |d         ¦  «        D ]-}|                     | j        |d         z  | j        z   ¦  «         Œ.d                     |¦  «        dz   S )Nr   rþ   rÿ   rl   z<s>)r  rZ   r¦   r§   rp   )r¬   r  rý   r�   Úimage_tokens_with_newlinesrá   s         r   r	  z!FuyuProcessor.replace_image_token®  sv   € à%'Ð"Ý�q˜& Ô/°¸Ô1DÑEÔEð 	qð 	qˆAØ&×-Ò-¨dÔ.>ÀÈÔATÑ.TÐW[ÔWoÑ.oÑpÔpÐpÐpð �wŠwÐ1Ñ2Ô2°UÑ:Ð:r   c                 ó
   — g d¢S )N)rø   rù   Úimage_sizesr¼   r   r°   s    r   rË   z FuyuProcessor.unused_input_names·  s   € àhÐhÐhÐhr   c           
      ó4  — i }|��ˆ|                      d¦  «        p| j        j        }|d         |d         }}g }dgt          |¦  «        z  }|D �]&}	||	d         z  }
||	d         z  }t	          |
|¦  «        }t	          t          |	d         |z  ¦  «        |	d         ¦  «        }t	          t          |	d         |z  ¦  «        |	d         ¦  «        }| j                             t          j        ddd||¦  «        t          j	        ddd¦  «        t          j
        |gg¦  «        t          j
        |gg¦  «        ddd¬	¦  «        }|                     |d
         d         d         j        d         ¦  «         �Œ(|                     ||dœ¦  «         t          di |¤ŽS )a»  
        Computes the number of placeholder tokens needed for multimodal inputs with the given sizes.

        Args:
            image_sizes (`list[list[int]]`, *optional*):
                The input sizes formatted as (height, width) per each image.

        Returns:
            `MultiModalData`: A `MultiModalData` object holding number of tokens per each of the provided
            input modalities, along with other useful data.
        NÚsizer  r  rT   r   r   T)Úimage_inputÚimage_presentÚimage_unpadded_hÚimage_unpadded_wÚimage_placeholder_idr©   Úvariable_sizedÚimage_input_idsrÁ   )Únum_image_tokensÚnum_image_patchesr   )rØ   r¢   r  rY   r  ÚintÚpreprocess_with_tokenizer_infor‡   ÚzerosÚonesÚtensorrZ   r‰   Úupdater   )r¬   r  r�   Úvision_datar  Úpadded_heightÚpadded_widthr   r!  Ú
image_sizeÚheight_scale_factorÚwidth_scale_factorÚoptimal_scale_factorr  r  Úmodel_image_inputs                   r   Ú_get_num_multimodal_tokensz(FuyuProcessor._get_num_multimodal_tokens»  sÈ  € ð ˆØÑ"Ø—:’:˜fÑ%Ô%ÐB¨Ô)=Ô)BˆDØ*.¨x¬.¸$¸w¼-˜<ˆMà!ÐØ!" ¥c¨+Ñ&6Ô&6Ñ 6ÐØ)ð ^ñ ^�
Ø&3°jÀ´mÑ&CÐ#Ø%1°J¸q´MÑ%AÐ"Ý'*Ð+>Ð@RÑ'SÔ'SÐ$å#&¥s¨:°a¬=Ð;OÑ+OÑ'PÔ'PÐR\Ð]^ÔR_Ñ#`Ô#`Ð Ý#&¥s¨:°a¬=Ð;OÑ+OÑ'PÔ'PÐR\Ð]^ÔR_Ñ#`Ô#`Ð ð %)Ô$8×$WÒ$WÝ %¤¨A¨q°!°]ÀLÑ QÔ QÝ"'¤*¨Q°°1Ñ"5Ô"5Ý%*¤\Ð4DÐ3EÐ2FÑ%GÔ%GÝ%*¤\Ð4DÐ3EÐ2FÑ%GÔ%GØ)*Ø%&Ø#'ð %Xñ %ô %Ð!ð !×'Ò'Ð(9Ð:KÔ(LÈQÔ(OÐPQÔ(RÔ(XÐY[Ô(\Ñ]Ô]Ð]Ñ]Ø×ÒÐ4DÐ[lÐmÐmÑnÔnÐnÝÐ,Ð, Ð,Ð,Ð,r   c                 óÖ  ‡ ‡‡	— dˆ fd„	Š	ˆ fd„Šˆˆ	ˆ fd„}ˆˆ	ˆ fd„}|€6‰ j         j        d         ‰ j         j        d         fft          |¦  «        z  }n |j        d         d	k    rt	          d
¦  «        ‚t          |¦  «        t          |¦  «        k    rt	          d¦  «        ‚g }t          ||¦  «        D ]2\  }} |||¦  «        } |||¦  «        }|                     |¦  «         Œ3|S )a¦  
        Transforms raw coordinates detected by [`FuyuForCausalLM`] to the original images' coordinate space.
        Coordinates will be returned in "box" format, with the following pattern:
            `<box>top, left, bottom, right</box>`

        Point coordinates are not supported yet.

        Args:
            outputs ([`GenerateOutput`]):
                Raw outputs from `generate`.
            target_sizes (`torch.Tensor`, *optional*):
                Tensor of shape (batch_size, 2) where each entry is the (height, width) of the corresponding image in
                the batch. If set, found coordinates in the output sequence are rescaled to the target sizes. If left
                to None, coordinates will not be rescaled.

        Returns:
            `GenerateOutput`: Same output type returned by `generate`, with output token ids replaced with
                boxed and possible rescaled coordinates.
        Nc                 ó®   •— | \  }}|€%‰j         j        d         }‰j         j        d         }n|\  }}||k    r||k    rdS t          ||z  ||z  ¦  «        S )Nr  r  g      ð?)r¢   r  r  )Úoriginal_sizeÚtarget_sizer  r  Ú
max_heightÚ	max_widthr¬   s         €r   Úscale_factor_to_fitzGFuyuProcessor.post_process_box_coordinates.<locals>.scale_factor_to_fitû  sr   ø€ Ø)‰MˆF�EØÐ"Ø!Ô1Ô6°xÔ@�
Ø Ô0Ô5°gÔ>�	�	à(3Ñ%�
˜IØ˜	Ò!Ð! f°
Ò&:Ð&:Ø�sÝ�z FÑ*¨I¸Ñ,=Ñ>Ô>Ð>r   c                 ó`  •— ‰j                              |¦  «        }‰j                              |¦  «        }| |k                         d¬¦  «        d         }| |k                         d¬¦  «        d         }t          j        |¦  «        r$t          j        |¦  «        r|d         |d         fS dS )NT©Úas_tupler   ©NN)rž   Úconvert_tokens_to_idsÚnonzeror‡   Úany)rv   Ústart_tokenÚ	end_tokenÚstart_idÚend_idÚstarting_positionsÚending_positionsr¬   s          €r   Úfind_delimiters_pairzHFuyuProcessor.post_process_box_coordinates.<locals>.find_delimiters_pair  s±   ø€ Ø”~×;Ò;¸KÑHÔHˆHØ”^×9Ò9¸)ÑDÔDˆFà"(¨HÒ"4×!=Ò!=ÀtÐ!=Ñ!LÔ!LÈQÔ!OÐØ &¨&Ò 0×9Ò9À4Ð9ÑHÔHÈÔKÐåŒyÐ+Ñ,Ô,ð Dµ´Ð;KÑ1LÔ1Lð DØ*¨1Ô-Ð/?ÀÔ/BÐCÐCØ�<r   c           
      óz  •‡—  ‰| t           t          ¦  «        x}dk    �r|\  }}||dz   k    rŒ-‰j                             | |dz   |…         ¦  «        } ‰|¦  «        Šˆfd„|D ¦   «         \  }}}}	dt          › |› d|› d|› d|	› t
          › �
}
‰j                             |
¦  «        dd …         }
‰j                             |
¦  «        }
t          j	        |
¦  «         
                    | ¦  «        }
t          j        | d |…         |
| |dz   d …         gd¦  «        }  ‰| t           t          ¦  «        x}dk    �°| S )Nr;  é   rT   c                 óT   •— g | ]$}d t          t          |¦  «        ‰z  ¦  «        z  ‘Œ%S ©r4   ©r"  re   ©rh   Úcr¿   s     €r   rj   zWFuyuProcessor.post_process_box_coordinates.<locals>.tokens_to_boxes.<locals>.<listcomp>  s1   ø€ Ð+VÐ+VÐ+VÈ!¨AµµE¸!±H´H¸uÑ4DÑ0EÔ0EÑ,EÐ+VÐ+VÐ+Vr   ú ú, r   )rL   rN   rž   Úconvert_ids_to_tokensrK   rM   Útokenizer<  r‡   r&  Útorˆ   )rv   r3  ÚpairÚstartÚendÚcoordsr:   r;   r<   r=   Úreplacementr¿   rE  r7  r¬   s              @€€€r   Útokens_to_boxeszCFuyuProcessor.post_process_box_coordinates.<locals>.tokens_to_boxes  sˆ  øø€ Ø/Ð/°Õ8NÕPgÑhÔhÐh�4ð nò ñ ð "‘
��sØ˜% !™)Ò#Ð#Øð œ×=Ò=¸fÀUÈQÁYÐQTÀ_Ô>UÑVÔV�ð ,Ð+¨MÑ:Ô:�Ø+VÐ+VÐ+VÐ+VÈvÐ+VÑ+VÔ+VÑ(��T˜6 5ð nÕ"5Ðm°sÐmÐm¸dÐmÐmÀfÐmÐmÐPUÐmÕWkÐmÐm�Ø"œn×5Ò5°kÑBÔBÀ1À2À2ÔF�Ø"œn×BÒBÀ;ÑOÔO�Ý#œl¨;Ñ7Ô7×:Ò:¸6ÑBÔB�åœ F¨6¨E¨6¤N°KÀÈÈaÉÈ	È	ÔARÐ#SÐUVÑWÔW�ð+ 0Ð/°Õ8NÕPgÑhÔhÐh�4ð nò ñ ð, ˆMr   c                 ój  •‡	—  ‰
| t           t          ¦  «        x}dk    �r|\  }}||dz   k    rŒ-‰j                             | |dz   |…         ¦  «        } ‰|¦  «        Š	ˆ	fd„|D ¦   «         \  }}dt          › |› d|› t
          › �}‰j                             |¦  «        dd …         }‰j                             |¦  «        }t          j	        |¦  «         
                    | ¦  «        }t          j        | d |…         || |dz   d …         gd¦  «        }  ‰
| t           t          ¦  «        x}dk    �°| S )Nr;  r   rT   c                 óT   •— g | ]$}d t          t          |¦  «        ‰z  ¦  «        z  ‘Œ%S rI  rJ  rK  s     €r   rj   zXFuyuProcessor.post_process_box_coordinates.<locals>.tokens_to_points.<locals>.<listcomp>8  s1   ø€ ÐBÐBÐB°a˜�C¥ a¡¤¨5Ñ 0Ñ1Ô1Ñ1ÐBÐBÐBr   rM  rN  r   )rH   rJ   rž   rO  rG   rI   rP  r<  r‡   r&  rQ  rˆ   )rv   r3  rR  rS  rT  rU  r.   r/   rV  r¿   rE  r7  r¬   s            @€€€r   Útokens_to_pointszDFuyuProcessor.post_process_box_coordinates.<locals>.tokens_to_points*  sr  øø€ Ø/Ð/°Õ8OÕQiÑjÔjÐj�4ð pò ñ ð "‘
��sØ˜% !™)Ò#Ð#Øð œ×=Ò=¸fÀUÈQÁYÐQTÀ_Ô>UÑVÔV�ð ,Ð+¨MÑ:Ô:�ØBÐBÐBÐB¸6ÐBÑBÔB‘��1ð XÕ"6ÐW¸ÐWÐW¸QÐWÕ@UÐWÐW�Ø"œn×5Ò5°kÑBÔBÀ1À2À2ÔF�Ø"œn×BÒBÀ;ÑOÔO�Ý#œl¨;Ñ7Ô7×:Ò:¸6ÑBÔB�åœ F¨6¨E¨6¤N°KÀÈÈaÉÈ	È	ÔARÐ#SÐUVÑWÔW�ð+ 0Ð/°Õ8OÕQiÑjÔjÐj�4ð pò ñ ð, ˆMr   r  r  rT   r4   zTEach element of target_sizes must contain the size (h, w) of each image of the batchzCMake sure that you pass in as many target sizes as output sequencesr!   )r¢   r  rY   r‰   ro   rz   rZ   )
r¬   ÚoutputsÚtarget_sizesrW  rZ  ÚresultsÚseqr  rE  r7  s
   `       @@r   Úpost_process_box_coordinatesz*FuyuProcessor.post_process_box_coordinatesæ  sh  øøø€ ð*		?ð 		?ð 		?ð 		?ð 		?ð 		?ð		 ð 		 ð 		 ð 		 ð 		 ð	ð 	ð 	ð 	ð 	ð 	ð 	ð2	ð 	ð 	ð 	ð 	ð 	ð 	ð2 ÐØ!Ô1Ô6°xÔ@À$ÔBVÔB[Ð\cÔBdÐeÐgÕjmÐnuÑjvÔjvÑvˆLˆLØÔ Ô" aÒ'Ð'ÝÐsÑtÔtÐtåˆw‰<Œ<�3˜|Ñ,Ô,Ò,Ð,ÝÐbÑcÔcÐcàˆÝ˜W lÑ3Ô3ð 	 ð 	 ‰IˆC�Ø!�/ # tÑ,Ô,ˆCØ"Ð" 3¨Ñ-Ô-ˆCØ�NŠN˜3ÑÔÐÐàˆr   Tc                 ó†  ‡	— | j                              t          ¦  «        Š	ˆ	fd„|D ¦   «         }t          d„ |D ¦   «         ¦  «        }t	          j        t          |¦  «        |f| j         j        ¦  «        }t          |¦  «        D ]-\  }}t	          j	        |¦  «        ||dt          |¦  «        …f<   Œ. | j
        |fd|i|¤ŽS )aú  
        Post-processes the output of `FuyuForConditionalGeneration` to only return the text output.

        Args:
            generated_outputs (`torch.Tensor` or `np.ndarray`):
                The output of the model. The output is expected to be a tensor of shape `(batch_size, sequence_length)`
                containing the token ids of the generated sequences.
            skip_special_tokens (`bool`, *optional*, defaults to `True`):
                Whether or not to remove special tokens in the output. Argument passed to the tokenizer's `batch_decode` method.
            **kwargs:
                Additional arguments to be passed to the tokenizer's `batch_decode method`.

        Returns:
            `list[str]`: The decoded text output.
        c                 óf   •— g | ]-}||‰k                          d ¬¦  «        d         dz   d…         ‘Œ.S )Tr9  r   rT   N)r=  )rh   r^  Úbeginning_of_answers     €r   rj   zAFuyuProcessor.post_process_image_text_to_text.<locals>.<listcomp>f  sR   ø€ ð %
ð %
ð %
ØRUˆC�Ð+Ò+×4Ò4¸dÐ4ÑCÔCÀAÔFÈÑJÐLÐLÔMð%
ð %
ð %
r   c              3   ó4   K  — | ]}t          |¦  «        V — Œd S r!   )rY   )rh   r^  s     r   ú	<genexpr>z@FuyuProcessor.post_process_image_text_to_text.<locals>.<genexpr>i  s(   è è € ÐDÐD 3•c˜#‘h”hÐDÐDÐDÐDÐDÐDr   NÚskip_special_tokens)rž   r<  rð   Úmaxr‡   ÚfullrY   r¥   rX   r&  Úbatch_decode)
r¬   Úgenerated_outputsre  r�   Úunpadded_output_sequencesÚmax_lenÚpadded_output_sequencesr^   r^  rb  s
            @r   Úpost_process_image_text_to_textz-FuyuProcessor.post_process_image_text_to_textS  sò   ø€ ð  #œn×BÒBÕC]Ñ^Ô^Ðð%
ð %
ð %
ð %
ØYjð%
ñ %
ô %
Ð!õ ÐDÐDÐ*CÐDÑDÔDÑDÔDˆå"'¤*­cÐ2KÑ.LÔ.LÈgÐ-VÐX\ÔXfÔXsÑ"tÔ"tÐÝÐ 9Ñ:Ô:ð 	Gð 	G‰FˆAˆsÝ5:´\À#Ñ5FÔ5FÐ# A z­¨S©¬ z MÑ2Ð2à ˆtÔ Ð!8ÐlÐlÐNaÐlÐekÐlÐlÐlr   c                 ó@   — | j         j        }t          |dgz   ¦  «        S )Nr   )rž   r—   rî   )r¬   Útokenizer_input_namess     r   r—   zFuyuProcessor.model_input_namesq  s$   € à $¤Ô @ÐÝÐ)¨_Ð,=Ñ=Ñ>Ô>Ð>r   )rl   r;  r!   )T)r   r   r   r   Úvalid_processor_kwargsÚclassmethodrŸ   r«   Úpropertyrî   r"  r±   r   r   rn   r   r   r   rë   rÔ   r	   rÕ   r×   Údictr	  rË   r0  r_  rm  r—   Ú__classcell__)r®   s   @r   r‘   r‘   Ê   sÄ  ø€ € € € € ð 1ÐààJLðð ð ñ „[ðð	Oð 	Oð 	Oð 	Oð 	Oð ð<  c¤ð <ð <ð <ñ „Xð<ð ð %)ØGKðorð orà˜TÑ!ðorð �D˜”I‰o 	Ñ)Ð,=Ñ=ÀÑDðorð Ð,Ô-ð	orð
 
ðorð orð orñ „^ðorðf %)ØZ^ð1ð 1à˜TÑ!ð1ð Ð+Ñ+¨d°9¬oÑ=ÀÐEVÔ@WÑWð1ð 1ð 1ð 1ðD 8<ØZ^ð	Lð 	Là˜T *Ô-Ñ-°Ñ4ð	Lð Ð+Ñ+¨d°9¬oÑ=ÀÐEVÔ@WÑWð	Lð Ð)Ô*ð		Lð 	Lð 	Lð 	Lð 	Lð 	Lð!4 jð !4ð !4ð !4ð !4ðF;°ð ;Àð ;ÐSVð ;ð ;ð ;ð ;ð ði D¨¤Ið ið ið iñ „Xðið)-ð )-ð )-ð )-ðVkð kð kð kðZmð mð mð mð< ð?ð ?ñ „Xð?ð ?ð ?ð ?ð ?r   r‘   )8Ú__doc__r  rU   Útypingr   Únumpyr#   Úimage_processing_utilsr   Úimage_utilsr   r   Úprocessing_utilsr   r	   r
   r   Útokenization_utils_baser   r   Úutilsr   r   r   r   Úutils.import_utilsr   Ú
get_loggerr   rñ   r‡   rK   rM   rG   rI   rL   rN   rH   rJ   rð   r   r)   r-   re   rî   r"  r9   rC   rn   rO   r`   rw   r   r�   r‘   Ú__all__r   r   r   ú<module>r€     sµ  ððð ð €€€Ø 	€	€	€	Ø Ð Ð Ð Ð Ð à Ð Ð Ð à 2Ð 2Ð 2Ð 2Ð 2Ð 2Ø ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?ðð ð ð ð ð ð ð ð ð ð ð ð DÐ CÐ CÐ CÐ CÐ CÐ CÐ CØ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SØ *Ð *Ð *Ð *Ð *Ð *ð 
ˆÔ	˜HÑ	%Ô	%€ð ÐÑÔð Ø€L€L€Lð Ð ØÐ Ø Ð Ø"Ð à!Ð Ø"Ð Ø"Ð Ø#Ð Ø%Ð ðð ð ð ð Ð*°%ð ñ ô ð ð@ð @ð @ð
@ð @ð @ð ¨ð  °%ð  Àuð  ÐQUÐVYÔQZð  ð  ð  ð  ðBØ	ðBØðBØ%*ðBØ38ðBØHMðBà	ˆ#„YðBð Bð Bð Bð°ð ¸ð ð ð ð ð¸ð Àð ð ð ð ð2G ð G°5ð G¸Sð Gð Gð Gð GðD  S¤	ð ¸$¸~Ô:Nð ÐSWÐX[ÔS\ð ð ð ð ð6Ø  T¨#¤Y¤°Ð ?Ô@ðà ðð �t˜NÔ+Ô,ðð ð	ð
 
ˆ.Ôðð ð ð ð" 
€�;ÐÑÔØðh?ð h?ð h?ð h?ð h?�Nñ h?ô h?ñ „ñ  Ôðh?ðV Ð
€€€r   