§
    ’©ºj·N  ã                   óš  — d dl Z d dlZd dlZd dlmZ d dlmZ d dlmZm	Z	m
Z
 d dlZd dlmZmZ d dlmZ d dlmZmZ d dlmZmZmZmZmZmZ d	Zed	         Zd
Ze j                              ej        ¦   «         d¦  «        Z dddœZ! G d„ d¦  «        Z"	 	 	 	 	 ddee#         deee#                  de$de%dede%de
ee&                  defd„Z'dS )é    N)ÚSequence)Úpartial)ÚAnyÚClassVarÚOptional)ÚTensorÚtensor)ÚLiteral)Ú_bleu_score_computeÚ_bleu_score_update)Ú_IPADIC_AVAILABLEÚ_MECAB_AVAILABLEÚ_MECAB_KO_AVAILABLEÚ_MECAB_KO_DIC_AVAILABLEÚ_REGEX_AVAILABLEÚ_SENTENCEPIECE_AVAILABLE©	ÚnoneÚ13aÚzhÚintlÚcharúja-mecabúko-mecabÚ	flores101Ú	flores200))u   ã�€u   ä¶µ)u   ä¸€u   é¾¥)u   é¾¦u   é¾»)u   ï¤€u   ï¨­)u   ï¨°u   ï©ª)u   ï©°u   ï«™)u   â€€0u   â©­6)u   â¾€0u   â¾¡d)u   ï¼€u   ï¿¯)u   âº€u   â»¿)u   ã€€u   ã€¿)u   ã‡€u   ã‡¯)u   â¼€u   â¿Ÿ)u   â¿°u   â¿¿)u   ã„€u   ã„¯)u   ã† u   ã†¿)u   ï¸�u   ï¸Ÿ)u   ï¸°u   ï¹�)u   â˜€u   â›¿)u   âœ€u   âž¿)u   ãˆ€u   ã‹¿)u   ãŒ€u   ã�¿ztorchmetrics-floreszRhttps://dl.fbaipublicfiles.com/fairseq/models/flores/sacrebleu_tokenizer_spm.modelz)https://tinyurl.com/flores200sacrebleuspm©r   r   c                   óØ  — e Zd ZU dZ ej        d¦  «        df ej        d¦  «        df ej        d¦  «        df ej        d¦  «        dffZerHd	d
lZe                     d¦  «        dfe                     d¦  «        dfe                     d¦  «        dffZ	ddddddddddœ	Z
ee         ed<   d
d
dœZeeeee         f                  ed<   d5dededd
fd„Zd edee         fd!„Ze	 d5d"ed          d edededee         f
d#„¦   «         Zed"ed          d edefd$„¦   «         Zed%edefd&„¦   «         Zed"ed          d edefd'„¦   «         Zed"ed          d edefd(„¦   «         Zed"ed          d edefd)„¦   «         Zed"ed          d edefd*„¦   «         Z ed"ed          d edefd+„¦   «         Z!ed"ed          d edefd,„¦   «         Z"ed"ed          d edefd-„¦   «         Z#ed"ed          d ede$d         defd.„¦   «         Z%ed"ed          d edefd/„¦   «         Z&ed"ed          d edefd0„¦   «         Z'ed ededefd1„¦   «         Z(ed"ed          dedd
fd2„¦   «         Z)ed3e$d         dd
fd4„¦   «         Z*d
S )6Ú_SacreBLEUTokenizerz�Tokenizer used for SacreBLEU calculation.

    Source: https://github.com/mjpost/sacrebleu/tree/master/sacrebleu/tokenizers

    z([\{-\~\[-\` -\&\(-\+\:-\@\/])z \1 z([^0-9])([\.,])z\1 \2 z([\.,])([^0-9])z \1 \2z
([0-9])(-)r   Nz(\P{N})(\p{P})z(\p{P})(\P{N})z(\p{S})Ú_tokenize_baseÚ_tokenize_13aÚ_tokenize_zhÚ_tokenize_internationalÚ_tokenize_charÚ_tokenize_ja_mecabÚ_tokenize_ko_mecabÚ_tokenize_flores_101Ú_tokenize_flores_200r   Ú_TOKENIZE_FNr   Úsentencepiece_processorsFÚtokenizeÚ	lowercaseÚreturnc                 ó~   — |                       |¦  «         t          | | j        |         ¦  «        | _        || _        d S ©N)Ú_check_tokenizers_validityÚgetattrr)   Útokenize_fnr,   )Úselfr+   r,   s      úw/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/torchmetrics/functional/text/sacre_bleu.pyÚ__init__z_SacreBLEUTokenizer.__init__“   s;   € Ø×'Ò'¨Ñ1Ô1Ð1å" 4¨Ô):¸8Ô)DÑEÔEˆÔØ"ˆŒˆˆó    Úlinec                 ó†   — |                       |¦  «        }|                      || j        ¦  «                             ¦   «         S r/   )r2   Ú_lowerr,   Úsplit)r3   r7   Útokenized_lines      r4   Ú__call__z_SacreBLEUTokenizer.__call__™   s7   € Ø×)Ò)¨$Ñ/Ô/ˆØ�{Š{˜>¨4¬>Ñ:Ô:×@Ò@ÑBÔBÐBr6   Úclsc                 óÈ   — |                       |¦  «         t          | | j        |         ¦  «        } ||¦  «        }|                      ||¦  «                             ¦   «         S r/   )r0   r1   r)   r9   r:   )r=   r7   r+   r,   r2   r;   s         r4   r+   z_SacreBLEUTokenizer.tokenize�   s^   € ð 	×&Ò& xÑ0Ô0Ð0å˜c 3Ô#3°HÔ#=Ñ>Ô>ˆØ$˜ TÑ*Ô*ˆØ�zŠz˜.¨)Ñ4Ô4×:Ò:Ñ<Ô<Ð<r6   c                 ó–   — | j         D ]\  }}|                     ||¦  «        }Œd                     |                     ¦   «         ¦  «        S )z©Post-processing tokenizer for `13a` and `zh` tokenizers.

        Args:
            line: a segment to tokenize

        Return:
            the tokenized line

        ú )Ú_REGEXÚsubÚjoinr:   ©r=   r7   Ú_reÚrepls       r4   Ú_tokenize_regexz#_SacreBLEUTokenizer._tokenize_regexª   sH   € ð œð 	'ð 	'‰IˆC�Ø—7’7˜4 Ñ&Ô&ˆDˆDà�xŠx˜Ÿ
š
™œÑ%Ô%Ð%r6   Úucharc                 óD   ‡ — t          ˆ fd„t          D ¦   «         ¦  «        S )z­Check if character is chinese.

        Args:
            uchar: input char in unicode.

        Return:
            whether the input char is a Chinese character.

        c              3   ó>   •K  — | ]\  }}|‰cxk    o|k    nc V — Œd S r/   © )Ú.0ÚstartÚendrH   s      €r4   ú	<genexpr>z7_SacreBLEUTokenizer._is_chinese_char.<locals>.<genexpr>Å   sC   øè è € ÐIÐI©Z¨U°C�5˜EÐ(Ð(Ò(Ð( SÒ(Ð(Ð(Ð(ÐIÐIÐIÐIÐIÐIr6   )ÚanyÚ_UCODE_RANGES)rH   s   `r4   Ú_is_chinese_charz$_SacreBLEUTokenizer._is_chinese_charº   s(   ø€ õ ÐIÐIÐIÐI½=ÐIÑIÔIÑIÔIÐIr6   c                 ó   — |S )zœTokenizes an input line with the tokenizer.

        Args:
            line: a segment to tokenize

        Return:
            the tokenized line

        rK   ©r=   r7   s     r4   r    z"_SacreBLEUTokenizer._tokenize_baseÇ   s	   € ð ˆr6   c                 óp  — |                      dd¦  «        }|                      dd¦  «        }|                      dd¦  «        }d|v rX|                      dd¦  «        }|                      d	d¦  «        }|                      d
d¦  «        }|                      dd¦  «        }|                      d|› d�¦  «        S )zÒTokenizes a line using a relatively minimal tokenization that is equivalent to mteval-v13a, used by WMT.

        Args:
            line: input sentence

        Return:
            tokenized sentence

        z	<skipped>Ú z-
ú
r@   ú&z&quot;ú"z&amp;z&lt;ú<z&gt;ú>)ÚreplacerG   rT   s     r4   r!   z!_SacreBLEUTokenizer._tokenize_13aÔ   s®   € ð �|Š|˜K¨Ñ,Ô,ˆØ�|Š|˜E 2Ñ&Ô&ˆØ�|Š|˜D #Ñ&Ô&ˆà�$ˆ;ˆ;Ø—<’< ¨#Ñ.Ô.ˆDØ—<’< ¨Ñ-Ô-ˆDØ—<’< ¨Ñ,Ô,ˆDØ—<’< ¨Ñ,Ô,ˆDà×"Ò" ; t ; ; ;Ñ/Ô/Ð/r6   c                 ó¶   — |                      ¦   «         }d}|D ],}|                      |¦  «        r|dz  }||z  }|dz  }Œ'||z  }Œ-|                      |¦  «        S )ap  Tokenization of Chinese text.

        This is done in two steps: separate each Chinese characters (by utf-8 encoding) and afterwards tokenize the
        Chinese part (following the `13a` i.e. mteval tokenizer).
        Author: Shujian Huang huangsj@nju.edu.cn.

        Args:
            line: input sentence

        Return:
            tokenized sentence

        rV   r@   )ÚstriprR   rG   )r=   r7   Úline_in_charsr   s       r4   r"   z _SacreBLEUTokenizer._tokenize_zhì   s{   € ð �zŠz‰|Œ|ˆØˆàð 	&ð 	&ˆDØ×#Ò# DÑ)Ô)ð &Ø Ñ$�Ø Ñ%�Ø Ñ$��à Ñ%��à×"Ò" =Ñ1Ô1Ð1r6   c                 ó–   — | j         D ]\  }}|                     ||¦  «        }Œd                     |                     ¦   «         ¦  «        S )a¢  Tokenizes a string following the official BLEU implementation.

        See github.com/moses-smt/mosesdecoder/blob/master/scripts/generic/mteval-v14.pl#L954-L983

        In our case, the input string is expected to be just one line.
        We just tokenize on punctuation and symbols,
        except when a punctuation is preceded and followed by a digit
        (e.g. a comma/dot as a thousand/decimal separator).
        We do not recover escaped forms of punctuation such as &apos; or &gt;
        as these should never appear in MT system outputs (see issue #138)

        Note that a number (e.g., a year) followed by a dot at the end of
        sentence is NOT tokenized, i.e. the dot stays with the number because
        `s/(\\p{P})(\\P{N})/ $1 $2/g` does not match this case (unless we add a
        space after each sentence). However, this error is already in the
        original mteval-v14.pl and we want to be consistent with it.
        The error is not present in the non-international version,
        which uses `$norm_text = " $norm_text "`.

        Args:
            line: the input string to tokenize.

        Return:
            The tokenized string.

        r@   )Ú
_INT_REGEXrB   rC   r:   rD   s       r4   r#   z+_SacreBLEUTokenizer._tokenize_international  sH   € ð8 œð 	'ð 	'‰IˆC�Ø—7’7˜4 Ñ&Ô&ˆDˆDà�xŠx˜Ÿ
š
™œÑ%Ô%Ð%r6   c                 ó@   — d                      d„ |D ¦   «         ¦  «        S )z Tokenizes all the characters in the input line.

        Args:
            line: a segment to tokenize

        Return:
            the tokenized line

        r@   c              3   ó   K  — | ]}|V — Œd S r/   rK   )rL   r   s     r4   rO   z5_SacreBLEUTokenizer._tokenize_char.<locals>.<genexpr>4  s"   è è € Ð.Ð. ˜Ð.Ð.Ð.Ð.Ð.Ð.r6   )rC   rT   s     r4   r$   z"_SacreBLEUTokenizer._tokenize_char)  s%   € ð �xŠxÐ.Ð.¨Ð.Ñ.Ô.Ñ.Ô.Ð.r6   c                 óÂ   — ddl }ddl}|                     |j        dz   ¦  «        }|                     ¦   «         }|                     |¦  «                             ¦   «         S )zÀTokenizes a Japanese string line using MeCab morphological analyzer.

        Args:
            line: the input string to tokenize.

        Return:
            The tokenized string.

        r   Nú	 -Owakati)ÚipadicÚMeCabÚTaggerÚ
MECAB_ARGSr^   Úparse)r=   r7   rf   rg   Útaggers        r4   r%   z&_SacreBLEUTokenizer._tokenize_ja_mecab6  sZ   € ð 	ˆˆˆØˆˆˆà—’˜fÔ/°+Ñ=Ñ>Ô>ˆà�zŠz‰|Œ|ˆØ�|Š|˜DÑ!Ô!×'Ò'Ñ)Ô)Ð)r6   c                 óÂ   — ddl }ddl}|                     |j        dz   ¦  «        }|                     ¦   «         }|                     |¦  «                             ¦   «         S )zÅTokenizes a Korean string line using MeCab-korean morphological analyzer.

        Args:
            line: the input string to tokenize.

        Return:
            The tokenized string.

        r   Nre   )Úmecab_koÚmecab_ko_dicrh   ri   r^   rj   )r=   r7   rm   rn   rk   s        r4   r&   z&_SacreBLEUTokenizer._tokenize_ko_mecabI  s]   € ð 	ˆˆˆØÐÐÐà—’ Ô!8¸;Ñ!FÑGÔGˆà�zŠz‰|Œ|ˆØ�|Š|˜DÑ!Ô!×'Ò'Ñ)Ô)Ð)r6   c                 óü  — ddl }| j        |         €¹|                     ¦   «         | j        |<   t          j                             t          t          |                              d¦  «        d         ¦  «        }t          j         	                    |¦  «        s|  
                    |¦  «         | j        |                              |¦  «         d                     | j        |                              |¦  «        ¦  «        S )zëTokenizes a string line using sentencepiece tokenizer.

        Args:
            line: the input string to tokenize.
            tokenize: Tokenization technique to be used.

        Return:
            The tokenized string.

        r   Nú/éÿÿÿÿr@   )Úsentencepiecer*   ÚSentencePieceProcessorÚosÚpathrC   Ú_FLORES_LOCAL_DIRÚ_FLORES_MODELS_URLr:   ÚexistsÚdownload_flores_fileÚLoadÚEncodeAsPieces)r=   r7   r+   rr   Ú	file_paths        r4   Ú_tokenize_floresz$_SacreBLEUTokenizer._tokenize_flores\  sÖ   € ð 	ÐÐÐàÔ'¨Ô1Ð9Ø5B×5YÒ5YÑ5[Ô5[ˆCÔ(¨Ñ2åœŸšÕ%6Õ8JÈ8Ô8T×8ZÒ8ZÐ[^Ñ8_Ô8_Ð`bÔ8cÑdÔdˆIÝ”7—>’> )Ñ,Ô,ð 3Ø×(Ò(¨Ñ2Ô2Ð2àÔ(¨Ô2×7Ò7¸	ÑBÔBÐBà�xŠx˜Ô4°XÔ>×MÒMÈdÑSÔSÑTÔTÐTr6   c                 ó.   — |                       |d¦  «        S )zÕTokenizes a string line using sentencepiece tokenizer according to `FLORES-101`_ dataset.

        Args:
            line: the input string to tokenize.

        Return:
            The tokenized string.

        r   ©r}   rT   s     r4   r'   z(_SacreBLEUTokenizer._tokenize_flores_101w  ó   € ð ×#Ò# D¨+Ñ6Ô6Ð6r6   c                 ó.   — |                       |d¦  «        S )zÕTokenizes a string line using sentencepiece tokenizer according to `FLORES-200`_ dataset.

        Args:
            line: the input string to tokenize.

        Return:
            The tokenized string.

        r   r   rT   s     r4   r(   z(_SacreBLEUTokenizer._tokenize_flores_200„  r€   r6   c                 ó2   — |r|                       ¦   «         S | S r/   )Úlower)r7   r,   s     r4   r9   z_SacreBLEUTokenizer._lower‘  s   € àð 	 Ø—:’:‘<”<ÐØˆr6   c                 ó€  — || j         vr6t          dt          | j                              ¦   «         ¦  «        › �¦  «        ‚|dk    rt          st          d¦  «        ‚|dk    rt          rt          st          d¦  «        ‚|dk    rt          rt          st          d¦  «        ‚d|v rt          st          d	¦  «        ‚d
S d
S )z}Check if a supported tokenizer is chosen.

        Also check all dependencies of a given tokenizers are installed.

        z6Unsupported tokenizer selected. Please, choose one of r   zv`'intl'` tokenization requires that `regex` is installed. Use `pip install regex` or `pip install torchmetrics[text]`.r   z—`'ja-mecab'` tokenization requires that `MeCab` and `ipadic` are installed. Use `pip install mecab-python3 ipadic` or `pip install torchmetrics[text]`.r   z¡`'ko-mecab'` tokenization requires that `mecab_ko` and `mecab_ko_dic` are installed. Use `pip install mecab_ko mecab_ko_dic` or `pip install torchmetrics[text]`.Úfloresz›`'flores101' and 'flores200'` tokenizations require that `sentencepiece` is installed. Use `pip install sentencepiece` or `pip install torchmetrics[text]`.N)r)   Ú
ValueErrorÚlistÚkeysr   ÚModuleNotFoundErrorr   r   r   r   r   )r=   r+   s     r4   r0   z._SacreBLEUTokenizer._check_tokenizers_validity—  s  € ð ˜3Ô+Ð+Ð+ÝÐuÕVZÐ[^Ô[k×[pÒ[pÑ[rÔ[rÑVsÔVsÐuÐuÑvÔvÐvà�vÒÐÕ&6ÐÝ%ðPñô ð ð
 �zÒ!Ð!Õ+;Ð!Õ@QÐ!Ý%ð_ñô ð ð
 �zÒ!Ð!Õ+>Ð!ÕCZÐ!Ý%ð`ñô ð ð
 �xÐÐÕ(@ÐÝ%ðXñô ð ð  ÐÐÐr6   Ú
model_namec                 ó4  — ddl }ddl}t          j        t          d¬¦  «         t
          |          }t          j                             t          |                     d¦  «        d         ¦  «        }	 t          |d¦  «        5 }|j
                             |¦  «        5 }|                     |                     ¦   «         ¦  «         ddd¦  «         n# 1 swxY w Y   ddd¦  «         dS # 1 swxY w Y   dS # |j        $ r}t          d| › d	�¦  «        |‚d}~ww xY w)
zGDownload necessary files for `flores` tokenization via `sentencepiece`.r   NT)Úexist_okrp   rq   ÚwbzFailed to download z model.)ÚsslÚurllib.requestrt   Úmakedirsrv   rw   ru   rC   r:   ÚopenÚrequestÚurlopenÚwriteÚreadÚSSLErrorÚOSError)rŠ   rŽ   ÚurllibÚ	model_urlr|   Úout_fileÚremote_fileÚes           r4   ry   z(_SacreBLEUTokenizer.download_flores_file¹  sž  € ð 	ˆ
ˆ
ˆ
ØÐÐÐå
ŒÕ%°Ð5Ñ5Ô5Ð5å& zÔ2ˆ	Ý”G—L’LÕ!2°I·O²OÀCÑ4HÔ4HÈÔ4LÑMÔMˆ	ð	LÝ�i Ñ&Ô&ð 3¨(°F´N×4JÒ4JÈ9Ñ4UÔ4Uð 3ÐYdØ—’˜{×/Ò/Ñ1Ô1Ñ2Ô2Ð2ð3ð 3ð 3ñ 3ô 3ð 3ð 3ð 3ð 3ð 3ð 3øøøð 3ð 3ð 3ð 3ð 3ð 3ð 3ñ 3ô 3ð 3ð 3ð 3ð 3ð 3ð 3ð 3øøøð 3ð 3ð 3ð 3ð 3ð 3øàŒ|ð 	Lð 	Lð 	LÝÐC°
ÐCÐCÐCÑDÔDÈ!ÐKøøøøð	Løøøs`   Á0C4 Â C'Â(CÃC'ÃC	ÃC'ÃC	ÃC'ÃC4 Ã'C+Ã+C4 Ã.C+Ã/C4 Ã4
DÃ>DÄD)F)+Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚreÚcompilerA   r   Úregexra   r)   r   ÚdictÚ__annotations__r*   Ústrr   r   Ú_TokenizersLiteralÚboolr5   r   r<   ÚclassmethodÚtyper+   rG   ÚstaticmethodrR   r    r!   r"   r#   r$   r%   r&   r
   r}   r'   r(   r9   r0   ry   rK   r6   r4   r   r   c   sŠ  € € € € € € ðð ð 
ˆŒÐ5Ñ	6Ô	6¸Ð@à	ˆŒÐ&Ñ	'Ô	'¨Ð3à	ˆŒÐ&Ñ	'Ô	'¨Ð3à	ˆŒ�MÑ	"Ô	" IÐ.ð€Fð ð 

Øˆˆˆð �]Š]Ð,Ñ-Ô-¨yÐ9à�]Š]Ð,Ñ-Ô-¨yÐ9à�]Š]˜:Ñ&Ô&¨Ð0ð
ˆ
ð !ØØØ)Ø Ø(Ø(Ø+Ø+ð
$ð 
$€L�(˜4”.ð 
ð 
ñ 
ð RVÐdhÐCiÐCiÐ˜h t¨C°¸#´Ð,>Ô'?Ô@ÐiÐiÑið#ð #Ð!3ð #Àð #ÐQUð #ð #ð #ð #ðC˜Sð C X¨c¤]ð Cð Cð Cð Cð ð
  ð	
=ð 
=ØÐ'Ô(ð
=àð
=ð %ð
=ð ð	
=ð
 
�#Œð
=ð 
=ð 
=ñ „[ð
=ð ð&˜TÐ"7Ô8ð &Àð &Èð &ð &ð &ñ „[ð&ð ð
J ð 
J¨ð 
Jð 
Jð 
Jñ „\ð
Jð ð
˜DÐ!6Ô7ð 
¸sð 
Àsð 
ð 
ð 
ñ „[ð
ð ð0˜4Ð 5Ô6ð 0¸cð 0Àcð 0ð 0ð 0ñ „[ð0ð. ð2˜$Ð4Ô5ð 2¸Sð 2ÀSð 2ð 2ð 2ñ „[ð2ð6 ð& TÐ*?Ô%@ð &Èð &ÐPSð &ð &ð &ñ „[ð&ð@ ð
/˜DÐ!6Ô7ð 
/¸sð 
/Àsð 
/ð 
/ð 
/ñ „[ð
/ð ð* Ð%:Ô ;ð *À3ð *È3ð *ð *ð *ñ „[ð*ð$ ð* Ð%:Ô ;ð *À3ð *È3ð *ð *ð *ñ „[ð*ð$ ðUØÐ'Ô(ðUØ03ðUØ?FÐG_Ô?`ðUà	ðUð Uð Uñ „[ðUð4 ð
7 $Ð'<Ô"=ð 
7ÀSð 
7ÈSð 
7ð 
7ð 
7ñ „[ð
7ð ð
7 $Ð'<Ô"=ð 
7ÀSð 
7ÈSð 
7ð 
7ð 
7ñ „[ð
7ð ð�Sð  Tð ¨cð ð ð ñ „\ðð
 ð¨Ð-BÔ(Cð ÐOað Ðfjð ð ð ñ „[ððB ðL¨Ð1IÔ)Jð LÈtð Lð Lð Lñ „\ðLð Lð Lr6   r   é   Fr   ÚpredsÚtargetÚn_gramÚsmoothr+   r,   Úweightsr-   c           
      óD  — t          | ¦  «        t          |¦  «        k    r/t          dt          | ¦  «        › dt          |¦  «        › �¦  «        ‚|�5t          |¦  «        |k    r"t          dt          |¦  «        › d|› �¦  «        ‚|€	d|z  g|z  }t          j        |¦  «        }t          j        |¦  «        }t	          d¦  «        }	t	          d¦  «        }
t          t          j        ||¬¦  «        }t          | ||||	|
||¦  «        \  }	}
t          |	|
|||||¦  «        S )a^  Calculate `BLEU score`_ [1] of machine translated text with one or more references.

    This implementation follows the behaviour of SacreBLEU [2] implementation from https://github.com/mjpost/sacrebleu.

    .. note::
        In the original SacreBLEU, references are passed as a list of reference sets (grouped by reference index).
        In TorchMetrics, references are passed grouped per prediction (each prediction has its own list of references).

        For example::

            # Predictions
            preds = ['The dog bit the man.', "It wasn't surprising.", 'The man had just bitten him.']

            # Original SacreBLEU:
            refs = [
                ['The dog bit the man.', 'It was not unexpected.', 'The man bit him first.'], # First set
                ['The dog had bit the man.', 'No one was surprised.', 'The man had bitten the dog.'], # Second set
            ]

            # TorchMetrics SacreBLEU:
            target = [
                ['The dog bit the man.', 'The dog had bit the man.'], # References for first prediction
                ['It was not unexpected.', 'No one was surprised.'], # References for second prediction
                ['The man bit him first.', 'The man had bitten the dog.'], # References for third prediction
            ]

    Args:
        preds: An iterable of machine translated corpus
        target: An iterable of iterables of reference corpus
        n_gram: Gram value ranged from 1 to 4
        smooth: Whether to apply smoothing - see [2]
        tokenize: Tokenization technique to be used. Choose between ``'none'``, ``'13a'``, ``'zh'``, ``'intl'``,
            ``'char'``, ``'ja-mecab'``, ``'ko-mecab'``, ``'flores101'`` and ``'flores200'``.
        lowercase: If ``True``, BLEU score over lowercased text is calculated.
        weights:
            Weights used for unigrams, bigrams, etc. to calculate BLEU score.
            If not provided, uniform weights are used.

    Return:
        Tensor with BLEU Score

    Raises:
        ValueError: If ``preds`` and ``target`` corpus have different lengths.
        ValueError: If a length of a list of weights is not ``None`` and not equal to ``n_gram``.

    Example:
        >>> from torchmetrics.functional.text import sacre_bleu_score
        >>> preds = ['the cat is on the mat']
        >>> target = [['there is a cat on the mat', 'a cat is on the mat']]
        >>> sacre_bleu_score(preds, target)
        tensor(0.7598)

    References:
        [1] BLEU: a Method for Automatic Evaluation of Machine Translation by Papineni,
        Kishore, Salim Roukos, Todd Ward, and Wei-Jing Zhu `BLEU`_

        [2] A Call for Clarity in Reporting BLEU Scores by Matt Post.

        [3] Automatic Evaluation of Machine Translation Quality Using Longest Common Subsequence
        and Skip-Bigram Statistics by Chin-Yew Lin and Franz Josef Och `Machine Translation Evolution`_

    zCorpus has different size z != Nz5List of weights has different weights than `n_gram`: g      ð?g        )r+   r,   )
Úlenr†   ÚtorchÚzerosr	   r   r   r+   r   r   )r­   r®   r¯   r°   r+   r,   r±   Ú	numeratorÚdenominatorÚ	preds_lenÚ
target_lenr2   s               r4   Úsacre_bleu_scorerº   Ë  s3  € õN ˆ5�z„z•S˜‘[”[Ò Ð ÝÐSµc¸%±j´jÐSÐSÅcÈ&ÁkÄkÐSÐSÑTÔTÐTàÐ�s 7™|œ|¨vÒ5Ð5ÝÐkÕQTÐU\ÑQ]ÔQ]ÐkÐkÐciÐkÐkÑlÔlÐlØ€Ø˜‘<�. 6Ñ)ˆå”˜FÑ#Ô#€IÝ”+˜fÑ%Ô%€KÝ�s‘”€IÝ˜‘”€JåÕ-Ô6ÀÐU^Ð_Ñ_Ô_€KÝ.ØØØØØØØØñ	ô 	Ñ€Iˆzõ ˜y¨*°iÀÈfÐV]Ð_eÑfÔfÐfr6   )r¬   Fr   FN)(rt   r¡   ÚtempfileÚcollections.abcr   Ú	functoolsr   Útypingr   r   r   r´   r   r	   Útyping_extensionsr
   Ú!torchmetrics.functional.text.bleur   r   Útorchmetrics.utilities.importsr   r   r   r   r   r   ÚAVAILABLE_TOKENIZERSr§   rQ   ru   rC   Ú
gettempdirrv   rw   r   r¦   Úintr¨   Úfloatrº   rK   r6   r4   ú<module>rÆ      sD  ððP 
€	€	€	Ø 	€	€	€	Ø €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø Ð Ð Ð Ð Ð Ø *Ð *Ð *Ð *Ð *Ð *Ð *Ð *Ð *Ð *à €€€Ø  Ð  Ð  Ð  Ð  Ð  Ð  Ð  Ø %Ð %Ð %Ð %Ð %Ð %à UÐ UÐ UÐ UÐ UÐ UÐ UÐ Uðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð oÐ ØÐrÔsÐ ð€ð6 ”G—L’LÐ!4 Ô!4Ñ!6Ô!6Ð8MÑNÔNÐ ð fØ<ðð Ð ðeLð eLð eLð eLð eLñ eLô eLð eLðV ØØ#(ØØ)-ð`gð `gØ�CŒ=ð`gà�X˜c”]Ô#ð`gð ð`gð ð	`gð
 !ð`gð ð`gð �h˜u”oÔ&ð`gð ð`gð `gð `gð `gð `gð `gr6   