§
    ’©ºjõ.  ã                   óž  — d dl Z d dlZd dlmZmZ d dlmZmZmZm	Z	m
Z
 d dlZd dlmZ d dlmZmZ d dlmZ d dlmZmZ ererd dlZerd dlmZmZ d	ed
efd„Z	 d+deeef         de	e
eej        f                  d
eeef         fd„Zded	eded
eeef         fd„Z ded	ed
eeeef         fd„Z!	 	 	 	 d,de"e         dedede#de#de#d
eeeef         e	e         f         fd„Z$d-dede#d
e
ed f         fd!„Z%d"eded
dfd#„Z&	 d+d$e
eej'        f         de	e
eej        f                  d
ed%         fd&„Z( G d'„ d(e¦  «        Z) G d)„ d*e)¦  «        Z*dS ).é    N)ÚCounterÚdefaultdict)ÚTYPE_CHECKINGÚAnyÚCallableÚOptionalÚUnion)ÚTensor)Ú
DataLoaderÚDataset)Ú_cumsum)Ú_TQDM_AVAILABLEÚ_TRANSFORMERS_GREATER_EQUAL_4_4)ÚPreTrainedModelÚPreTrainedTokenizerBaseÚattention_maskÚreturnc                 óâ   — d| dd…df<   t          | dz
  d¬¦  «                             d¦  «        }d| t          j        |                      d¦  «        ¦  «                             ¦   «         |f<   | S )a  Process attention mask to be zero for special [CLS] and [SEP] tokens as they're not included in BERT score.

    Args:
        attention_mask: An attention mask to be returned, for example, by a `transformers` tokenizer.

    Return:
        A processed attention mask.

    r   Ngš™™™™™¹?éÿÿÿÿ©Údim)r   ÚargmaxÚtorchÚarangeÚsizeÚlong)r   Úsep_token_positions     ú„/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/torchmetrics/functional/text/helper_embedding_metric.pyÚ*_process_attention_mask_for_special_tokensr   !   sv   € ð €N�1�1�1�a�4Ñå  .°3Ñ"6¸RÐ@Ñ@Ô@×GÒGÈÑKÔKÐØVW€N•5”< × 3Ò 3°AÑ 6Ô 6Ñ7Ô7×<Ò<Ñ>Ô>Ð@RÐRÑSØÐó    ÚbatchÚdevicec                 ój  — t          | d                              d¦  «                             ¦   «                              ¦   «         ¦  «        }| d         dd…d|…f                              |¦  «        }| d         dd…d|…f                              |¦  «        }|                      ||dœ¦  «         | S )z—Trim model inputs.

    This function trims the model inputs to the longest sequence within the batch and put the input on the proper
    device.

    r   é   Ú	input_idsN©r%   r   )ÚintÚsumÚmaxÚitemÚtoÚupdate)r!   r"   Úmax_lenr%   r   s        r   Ú_input_data_collatorr.   3   s¯   € õ �%Ð(Ô)×-Ò-¨aÑ0Ô0×4Ò4Ñ6Ô6×;Ò;Ñ=Ô=Ñ>Ô>€GØ�kÔ" 1 1 1 h w h ;Ô/×2Ò2°6Ñ:Ô:€IØÐ+Ô,¨Q¨Q¨Q°°°¨[Ô9×<Ò<¸VÑDÔD€NØ	‡L‚L˜y¸NÐKÐKÑLÔLÐLØ€Lr    Úmodel_outputÚ
target_lenc                 ó�  — t          | j        ¦  «        }||d         z
  |d<   t          j        | t          j        || j        ¬¦  «                             | j        ¦  «        gd¬¦  «        } t          j        |d         |d         |j        ¬¦  «                             |j        ¦  «        }t          j        ||gd¬¦  «        }| |fS )z=Pad the model output and attention mask to the target length.é   )Údtyper   r   r$   )ÚlistÚshaper   ÚcatÚzerosr3   r+   r"   )r/   r   r0   Úzeros_shaper7   s        r   Ú_output_data_collatorr9   C   sÁ   € å�|Ô)Ñ*Ô*€KØ +¨a¤.Ñ0€K��NÝ”9Ø	•u”{ ;°lÔ6HÐIÑIÔI×LÒLÈ\ÔM`ÑaÔaÐbÐhiðñ ô €Lõ ŒK˜ Aœ¨°A¬¸nÔ>RÐSÑSÔS×VÒVÐWeÔWlÑmÔm€EÝ”Y °Ð6¸AÐ>Ñ>Ô>€NØ˜Ð'Ð'r    r%   c                 óz   — |                      d¦  «                             ¦   «         }| |         } ||         }| ||fS )z=Sort tokenized sentence from the shortest to the longest one.r$   )r(   Úargsort)r%   r   Úsorted_indicess      r   Ú_sort_data_according_lengthr=   O   sB   € à#×'Ò'¨Ñ*Ô*×2Ò2Ñ4Ô4€NØ˜.Ô)€IØ# NÔ3€NØ�n nÐ4Ð4r    é   TFÚtextÚ	tokenizerÚ
max_lengthÚ
truncationÚsort_according_lengthÚown_tokenizerc                 ó  — |s || d||d¬¦  «        }n3	  || |¦  «        }n%# t           $ r}t          d|› �¦  «        |‚d}~ww xY w|r&t          |d         |d         ¦  «        \  }}	}
||	dœ}n|d         |d         dœ}d}
||
fS )	a  Text pre-processing function using `transformers` `AutoTokenizer` instance.

    Args:
        text:
            An iterable of sentences.
        tokenizer:
            Either `AutoTokenizer` instance from `transformers` package, or a user's own tokenizer.
        max_length:
            A maximum sequence length.
        truncation:
            An indication of whether tokenized sequences should be padded only to the length of the longest sequence.
        sort_according_length:
            An indication of whether tokenized sequences should be sorted from shortest to longest. This is appropriate
            to do for leveraging dynamic padding during embedding calculation and thereby to hasten inference.
        own_tokenizer:
            An indication of whether a non-default user's own tokenizer is used.

    Return:
        A dictionary of tokenized sentences including input_ids and attention_mask.

    Raises:
        BaseException:
            If a tokenization with a user's own tokenizer is not successful.

    rA   Úpt)ÚpaddingrA   rB   Úreturn_tensorsz!Tokenization was not successful: Nr%   r   r&   )ÚBaseExceptionÚRuntimeErrorr=   )r?   r@   rA   rB   rC   rD   Útokenized_dataÚexr%   r   Úsorting_indicesÚ
input_dicts               r   Ú_preprocess_textrO   W   sñ   € ðB ð QØ"˜Ø˜,°:È*Ðeið
ñ 
ô 
ˆˆð	QØ&˜Y t¨ZÑ8Ô8ˆNˆNøÝð 	Qð 	Qð 	QÝÐGÀ2ÐGÐGÑHÔHÈbÐPøøøøð	Qøøøð ð Ý5PØ˜;Ô'¨Ð8HÔ)Iñ6
ô 6
Ñ2ˆ	�> ?ð $-ÀÐOÐOˆ
ˆ
à#1°+Ô#>ÐR`ÐaqÔRrÐsÐsˆ
Øˆà�Ð&Ð&s   •" ¢
A¬?¿AÚ
dataloaderÚverboseztqdm.auto.tqdmc                 óF   — ddl }|r|j                              | ¦  «        n| S )a
  Wrap dataloader in progressbar if asked for.

    Function will return either the dataloader itself when `verbose = False`, or it wraps the dataloader with
    `tqdm.auto.tqdm`, when `verbose = True` to display a progress bar during the embeddings calculation.

    r   N)ÚtqdmÚauto)rP   rQ   rS   s      r   Ú_get_progress_barrU   Ž   s+   € ð €K€K€Kà)0Ð@ˆ4Œ9�>Š>˜*Ñ%Ô%Ð%°jÐ@r    Úoutputc           	      óÞ   — |j         dd…         \  }}t          | j         ¦  «        dk    p!| j         d         |k    p| j         d         |k    }|rt          d|› d|› d| j         › d	�¦  «        ‚dS )
z2Check if the shape of the user's own model output.Nr2   é   r   r$   zVThe model output must be `Tensor` of a shape `[batch_size, seq_len, model_dim]` i.e. [z, z. , `model_dim`], but got ú.)r5   ÚlenÚ
ValueError)rV   r%   ÚbsÚseq_lenÚinvalid_out_shapes        r   Ú_check_shape_of_model_outputr_   š   s¬   € à”/ " 1 "Ô%�K€BˆÝ˜FœLÑ)Ô)¨QÒ.Ðe°&´,¸q´/ÀRÒ2GÐeÈ6Ì<ÐXYÌ?Ð^eÒKeÐØð 
ÝðNØðNð NØ"ðNð NØ>D¼lðNð Nð Nñ
ô 
ð 	
ð
ð 
r    Úmodel_name_or_path)r   r   c                 óÀ   — ddl m}m} |                     | ¦  «        }|                     | ¦  «        }|                     ¦   «          |                     |¦  «         ||fS )ah  Load HuggingFace `transformers`' tokenizer and model. This function also handle a device placement.

    Args:
        model_name_or_path:
            A name or a model path used to load `transformers` pretrained model.
        device:
            A device to be used for calculation.

    Return:
        Initialized `transformers`' tokenizer and model.

    r   )ÚAutoModelForMaskedLMÚAutoTokenizer)Útransformersrb   rc   Úfrom_pretrainedÚevalr+   )r`   r"   rb   rc   r@   Úmodels         r   Ú_load_tokenizer_and_modelrh   ¥   so   € ð AÐ@Ð@Ð@Ð@Ð@Ð@Ð@à×-Ò-Ð.@ÑAÔA€IØ ×0Ò0Ð1CÑDÔD€EØ	‡J‚J�L„L€LØ	‡H‚HˆVÑÔÐØ�eÐÐr    c                   óR  — e Zd ZdZdedddfdee         dedede	ee         eee
geeeef         eeeef         ee         f         f         f         d	e
d
eeeef                  de
ddfd„Zdedeeef         fd„Zdefd„Zdeeef         fd„Zdefd„Zededefd„¦   «         ZdS )ÚTextDatasetzkPyTorch dataset class for storing tokenized sentences and other properties used for BERT score calculation.r>   FNr?   r@   rA   Úpreprocess_text_fnÚidfÚ
tokens_idfrB   r   c                 ó:  —  |||||¦  «        }t          |t          ¦  «        r|\  | _        | _        n|| _        | j        d         j        d         | _        t          |¦  «        | _        || _        i | _	        |r|�|n|  
                    ¦   «         | _	        dS dS )a�  Initialize text dataset class.

        Args:
            text: An iterable of sentences.
            tokenizer: `AutoTokenizer` instance from `transformers` package.
            max_length: A maximum sequence length.
            preprocess_text_fn: A function used for processing the input sentences.
            idf: An indication of whether calculate token inverse document frequencies to weight the model embeddings.
            tokens_idf: Inverse document frequencies (these should be calculated on reference sentences).
            truncation: An indication of whether tokenized sequences should be padded only to the length of the longest

        r%   r$   N)Ú
isinstanceÚtupler?   rM   r5   rA   rZ   Únum_sentencesrl   rm   Ú_get_tokens_idf)	Úselfr?   r@   rA   rk   rl   rm   rB   Ú_texts	            r   Ú__init__zTextDataset.__init__À   s¬   € ð0 #Ð" 4¨°JÀ
ÑKÔKˆÝ�e�UÑ#Ô#ð 	Ø.3Ñ+ˆDŒI�tÔ+Ð+àˆDŒIØœ) KÔ0Ô6°qÔ9ˆŒÝ  ™YœYˆÔØˆŒØˆŒØð 	_Ø,6Ð,B˜j˜jÈ×H\ÒH\ÑH^ÔH^ˆDŒOˆOˆOð	_ð 	_r    Úidxc                 óê   ‡ — ‰ j         d         |dd…f         }‰ j         d         |dd…f         }||dœ}‰ j        r7t          j        ˆ fd„|                     ¦   «         D ¦   «         ¦  «        }||d<   |S )zGGet the input ids and attention mask belonging to a specific datapoint.r%   Nr   r&   c                 ó*   •— g | ]}‰j         |         ‘ŒS © )rm   )Ú.0Ú	input_idxrs   s     €r   ú
<listcomp>z+TextDataset.__getitem__.<locals>.<listcomp>ê   s    ø€ Ð)iÐ)iÐ)iÈ¨$¬/¸)Ô*DÐ)iÐ)iÐ)ir    Úinput_ids_idf)r?   rl   r   ÚtensorÚtolist)rs   rv   r%   r   Úinputs_dictr}   s   `     r   Ú__getitem__zTextDataset.__getitem__ä   sŽ   ø€ à”I˜kÔ*¨3°°°¨6Ô2ˆ	ØœÐ#3Ô4°S¸!¸!¸!°VÔ<ˆØ$-ÀÐPÐPˆØŒ8ð 	9Ý!œLÐ)iÐ)iÐ)iÐ)iÐV_×VfÒVfÑVhÔVhÐ)iÑ)iÔ)iÑjÔjˆMØ+8ˆK˜Ñ(ØÐr    c                 ó   — | j         S )z.Return the number of sentences in the dataset.)rq   ©rs   s    r   Ú__len__zTextDataset.__len__î   s   € àÔ!Ð!r    c                 ó"  ‡ — t          ¦   «         }t          ‰ j        ‰ j        d         ¦  «        D ]}|                     |¦  «         Œt          ‰ j        ¦  «        }|                     ˆ fd„|                     ¦   «         D ¦   «         ¦  «         |S )zŸCalculate token inverse document frequencies.

        Return:
            A python dictionary containing inverse document frequencies for token ids.

        r%   c                 ó\   •— i | ](\  }}|t          j        ‰j        d z   |d z   z  ¦  «        “Œ)S )r$   ©ÚmathÚlogrq   )rz   rv   Ú
occurrencers   s      €r   ú
<dictcomp>z/TextDataset._get_tokens_idf.<locals>.<dictcomp>þ   sH   ø€ ð 
ð 
ð 
ÙKZÈ3ÐPZˆC•”˜4Ô-°Ñ1°jÀ1±nÑEÑFÔFð
ð 
ð 
r    )r   ÚmapÚ_set_of_tokensr?   r,   r   Ú_get_tokens_idf_default_valueÚitems)rs   Útoken_counterÚtokensrm   s   `   r   rr   zTextDataset._get_tokens_idfò   sª   ø€ õ ")¡¤ˆÝ˜$Ô-¨t¬y¸Ô/EÑFÔFð 	)ð 	)ˆFØ× Ò  Ñ(Ô(Ð(Ð(å'2°4Ô3UÑ'VÔ'Vˆ
Ø×Òð 
ð 
ð 
ð 
Ø^k×^qÒ^qÑ^sÔ^sð
ñ 
ô 
ñ 	ô 	ð 	ð Ðr    c                 ó@   — t          j        | j        dz   dz  ¦  «        S )z$Ensure `defaultdict` can be pickled.r$   r‡   rƒ   s    r   rŽ   z)TextDataset._get_tokens_idf_default_value  s   € åŒx˜Ô+¨aÑ/°1Ñ4Ñ5Ô5Ð5r    r%   c                 óD   — t          |                      ¦   «         ¦  «        S )zAReturn set of tokens from the `input_ids` :class:`~torch.Tensor`.)Úsetr   )r%   s    r   r�   zTextDataset._set_of_tokens  s   € õ �9×#Ò#Ñ%Ô%Ñ&Ô&Ð&r    )Ú__name__Ú
__module__Ú__qualname__Ú__doc__rO   r4   Ústrr   r'   r   Úboolr	   Údictr
   rp   r   Úfloatru   r�   r„   rr   rŽ   Ústaticmethodr”   r�   ry   r    r   rj   rj   ½   s¯  € € € € € ØuÐuð ð ØØ15Ø ð"_ð "_à�3Œið"_ð ð"_ð ð	"_ð
 %Ø�#ŒY˜˜S $Ð'¨¨t°C¸°KÔ/@À%ÈÈSÐRXÈ[ÔHYÐ[cÐdjÔ[kÐHkÔBlÐ/lÔ)mÐmô
ð"_ð ð"_ð ˜T # u *Ô-Ô.ð"_ð ð"_ð 
ð"_ð "_ð "_ð "_ðH˜sð  t¨C°¨KÔ'8ð ð ð ð ð"˜ð "ð "ð "ð "ð  c¨5 jÔ!1ð ð ð ð ð"6¨uð 6ð 6ð 6ð 6ð ð' &ð '¨Sð 'ð 'ð 'ñ „\ð'ð 'ð 'r    rj   c                   óP   — e Zd ZdZ	 	 d
dedededeeee	f                  ddf
d	„Z
dS )ÚTokenizedDatasetzHThe child class of `TextDataset` class used with already tokenized data.FNr%   r   rl   rm   r   c           	      ó’  — t          t          g d¢t          ||¦  «        ¦  «        ¦  «        }|                     d¦  «        | _        t          |¦  «        | _        t          | j        d         ¦  «        | _        | j        d         j	        d         | _
        || _        i | _        |r|�|n|                      ¦   «         | _        dS dS )aw  Initialize the dataset class.

        Args:
            input_ids: Input indexes
            attention_mask: Attention mask
            idf:
                An indication of whether calculate token inverse document frequencies to weight the model embeddings.
            tokens_idf: Inverse document frequencies (these should be calculated on reference sentences).

        )r%   r   rM   rM   r%   r$   N)r›   Úzipr=   ÚpoprM   r.   r?   rZ   rq   r5   rA   rl   rm   rr   )rs   r%   r   rl   rm   r?   s         r   ru   zTokenizedDataset.__init__  sÌ   € õ" ÝØBÐBÐBÝ+¨I°~ÑFÔFñô ñ
ô 
ˆð  $ŸxšxÐ(9Ñ:Ô:ˆÔÝ(¨Ñ.Ô.ˆŒ	Ý  ¤¨;Ô!7Ñ8Ô8ˆÔØœ) KÔ0Ô6°qÔ9ˆŒØˆŒØˆŒØð 	_Ø,6Ð,B˜j˜jÈ×H\ÒH\ÑH^ÔH^ˆDŒOˆOˆOð	_ð 	_r    )FN)r•   r–   r—   r˜   r
   rš   r   r›   r'   rœ   ru   ry   r    r   rŸ   rŸ     s…   € € € € € ØRÐRð Ø15ð_ð _àð_ð ð_ð ð	_ð
 ˜T # u *Ô-Ô.ð_ð 
ð_ð _ð _ð _ð _ð _r    rŸ   )N)r>   TTF)F)+rˆ   ÚosÚcollectionsr   r   Útypingr   r   r   r   r	   r   r
   Útorch.utils.datar   r   Útorchmetrics.utilities.datar   Útorchmetrics.utilities.importsr   r   rS   rd   r   r   r   r›   r™   r"   r.   r'   rp   r9   r=   r4   rš   rO   rU   r_   ÚPathLikerh   rj   rŸ   ry   r    r   ú<module>rª      s¢  ðð €€€Ø 	€	€	€	Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @à €€€Ø Ð Ð Ð Ð Ð Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0à /Ð /Ð /Ð /Ð /Ð /Ø [Ð [Ð [Ð [Ð [Ð [Ð [Ð [àð JØð ØˆˆˆØ&ð JØIÐIÐIÐIÐIÐIÐIÐIð¸vð È&ð ð ð ð ð& LPðð Ø��V�ÔðØ&.¨u°S¸%¼,Ð5FÔ/GÔ&Hðà	ˆ#ˆvˆ+Ôðð ð ð ð 	(¨ð 	(Àð 	(ÐTWð 	(Ð\aÐbhÐjpÐbpÔ\qð 	(ð 	(ð 	(ð 	(ð5¨6ð 5À6ð 5ÈeÐTZÐ\bÐdjÐTjÔNkð 5ð 5ð 5ð 5ð ØØ"&Øð4'ð 4'Ø
ˆsŒ)ð4'àð4'ð ð4'ð ð	4'ð
  ð4'ð ð4'ð ˆ4��V�Ô˜h vÔ.Ð.Ô/ð4'ð 4'ð 4'ð 4'ðn	Að 	A *ð 	A°tð 	AÈÈjÐZjÐNjÔHkð 	Að 	Að 	Að 	Að
¨ð 
¸Fð 
Àtð 
ð 
ð 
ð 
ð _cðð Ø˜c 2¤;Ð.Ô/ðØ9AÀ%ÈÈUÌ\ÐHYÔBZÔ9[ðà
Ð7Ô8ðð ð ð ð0M'ð M'ð M'ð M'ð M'�'ñ M'ô M'ð M'ð`!_ð !_ð !_ð !_ð !_�{ñ !_ô !_ð !_ð !_ð !_r    