§
    9²¸jÿ  ã                   ó¾  — d dl Z d dlmZ d dlmZ d dlmZmZ d dlm	Z	 d dl
Z
d dlmZ ddlmZmZmZ  G d	„ d
¦  «        Ze G d„ d¦  «        ¦   «         Zdedefd„Zdedededefd„Zd*dedededefd„Zdedededefd„Z	 d+de
j        dee         dee         deddf
d„Zd ed!ed"ed#ed$e	dee         fd%„Zd&e j        dee         fd'„Zdefd(„Z ed)„ ¦   «         Z!dS ),é    N)Úcontextmanager)Ú	dataclass)ÚceilÚlog2)ÚAny)ÚPretrainedConfigé   )ÚFutureRequestStateÚRequestStateÚRequestStatusc                   ó”   — e Zd ZdZdd„Zdd„Zdeedf         dej	        j
        dz  fd„Zdeedf         d	ej	        j
        ddfd
„ZdS )ÚCudaGraphBufferz`A dict for CUDA graphs with a special __del__ method to make sure the graphs are properly reset.ÚreturnNc                 ó   — i | _         d S ©N©Ú_storage)Úselfs    ú�/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/transformers/generation/continuous_batching/utils.pyÚ__init__zCudaGraphBuffer.__init__   s   € ØEGˆŒˆˆó    c                 ó†   — | j         r9| j                              ¦   «         \  }}|                     ¦   «          | j         °7d S d S r   )r   ÚpopitemÚreset)r   Ú_Úgraphs      r   Ú__del__zCudaGraphBuffer.__del__!   sP   € ØŒmð 	Ø”}×,Ò,Ñ.Ô.‰HˆAˆuØ�KŠK‰MŒMˆMð Œmð 	ð 	ð 	ð 	ð 	r   Úkey.c                 ó6   — | j                              |¦  «        S r   )r   Úget)r   r   s     r   Ú	get_graphzCudaGraphBuffer.get_graph&   s   € ØŒ}× Ò  Ñ%Ô%Ð%r   r   c                 ó   — || j         |<   d S r   r   )r   r   r   s      r   Ú	set_graphzCudaGraphBuffer.set_graph)   s   € Ø"ˆŒ�cÑÐÐr   )r   N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   ÚtupleÚintÚtorchÚcudaÚ	CUDAGraphr!   r#   © r   r   r   r      s°   € € € € € ØjÐjðHð Hð Hð Hðð ð ð ð
&˜U 3¨ 8œ_ð &°´Ô1EÈÑ1Lð &ð &ð &ð &ð#˜U 3¨ 8œ_ð #°U´ZÔ5Ið #Èdð #ð #ð #ð #ð #ð #r   r   c                   ó>   — e Zd ZU dZdZeed<   dZeed<   dZeed<   dS )ÚWorkloadHintszRA tiny dataclass containing hints to help choose good continuous batching defaultsr   Úmax_prompt_lengthÚmax_generated_lengthÚnum_requestsN)	r$   r%   r&   r'   r0   r)   Ú__annotations__r1   r2   r-   r   r   r/   r/   -   sJ   € € € € € € à\Ð\àÐ�sÐÐÑØ !Ð˜#Ð!Ð!Ñ!Ø€L�#ÐÐÑÐÐr   r/   Úconfigr   c                 ó   — | j         dv S )z:Checks if attention mask is needed for the given (config).)zpaged|eagerz
paged|sdpa)Ú_attn_implementation)r4   s    r   Úattn_mask_is_neededr7   6   s   € àÔ&Ð*GÐGÐGr   ÚsizeÚinterval_sizeÚ	max_valuec                 ól   — |dk    r|S | dk    rt          | |z  ¦  «        |z  n|}t          ||¦  «        S )zQReturn the smallest multiple of (interval_size) >= (size), capped at (max_value).r   )r   Úmin)r8   r9   r:   Úpaddeds       r   Úpad_to_intervalr>   ;   sG   € à˜ÒÐØÐØ;?À!º8¸8�T�$˜Ñ&Ñ'Ô'¨-Ñ7Ð7È€FÝˆv�yÑ!Ô!Ð!r   ÚvalueÚ	min_valuec                 ó¶   — t          | t          d|¦  «        ¦  «        } dt          t          t          | ¦  «        ¦  «        ¦  «        z  }t	          ||¦  «        S )z‘Return the smallest power of 2 >= (value), capped at (max_value). If a minimum value is provided, the value is at
    least padded to that value.r	   é   )Úmaxr)   r   r   r<   )r?   r:   r@   r=   s       r   Úpad_to_pow2rD   C   sM   € õ �•s˜1˜iÑ(Ô(Ñ)Ô)€EØ•#•d�4 ™;œ;Ñ'Ô'Ñ(Ô(Ñ(€FÝˆv�yÑ!Ô!Ð!r   ÚxÚ	divide_byÚalign_toc                 ód   — t          t          | |z  ¦  «        ¦  «        } | |z  r| || |z  z
  z  } | S r   )r)   r   )rE   rF   rG   s      r   Úaligned_dividerI   K   s?   € Ý�D��Y‘ÑÔÑ Ô €AØˆ8�|ð 'Ø	ˆX˜˜X™Ñ&Ñ&ˆØ€Hr   Úattention_maskÚcumulative_seqlens_qÚcumulative_seqlens_kÚsliding_windowc                 ój  — t          j        | j        ¦  «        j        }t	          t          |¦  «        dz
  ¦  «        D ]ô}||dz            ||         z
  }||dz            ||         z
  }||k     r|dk    r	||z
  dz   }nd}t          ||         ||dz            ¦  «        }	t          ||         ||dz            ¦  «        }
t          j        | d|	|
f         j        || j        | j	        ¬¦  «        }t          j
        ||¬¦  «        }|dk    r!||z
  |z
  }|t          j        ||¬¦  «        z  }|| d|	|
f<   ŒõdS )u  Builds an attention mask inplace using the cumulative seqlens of the query and key. If given a sliding window, it
    will also apply a sliding window mask on top. The attention mask is not boolean, it uses zeroes and -inf (or its
    equivalent) so it's more of an attention score bias tensor.
    The attention mask is a block-diagonal matrix, with each block an attention mask for a single query-key pair.
    Each of those block is built from a causal mask and, if there is a sliding window, a sliding window mask.

    An example is represented below, with seqlen_k = 8, seqlen_q = 4 and sliding_window = 6:

    CAUSAL MASK:

           â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–‘ â–‘ â–‘
           â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–‘ â–‘
           â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–‘
           â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ

    SLIDING WINDOW MASK:
         â”Œâ”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€ seqlen_k - seqlen_q - sliding_window = 8 - 4 - 6 = -2 offset to the left
       <â”€â”´â”€>
     â–‘ â–ˆ | â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ
     â–‘ â–‘ | â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ
     â–‘ â–‘ | â–‘ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ
     â–‘ â–‘ | â–‘ â–‘ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ

    ATTENTION MASK (sum of causal and sliding window masks):

           â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–‘ â–‘ â–‘
           â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–‘ â–‘
           â–‘ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–‘
           â–‘ â–‘ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ

    Another example with seqlen_k = 5, seqlen_q = 3 and sliding_window = 2:

    CAUSAL MASK:

           â–ˆ â–ˆ â–ˆ â–‘ â–‘
           â–ˆ â–ˆ â–ˆ â–ˆ â–‘
           â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ

    SLIDING WINDOW MASK:
         â”Œâ”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€ seqlen_k - seqlen_q - sliding_window = 5 - 3 - 2 = 0 offset to the left
        <â”´>
         | â–‘ â–ˆ â–ˆ â–ˆ â–ˆ
         | â–‘ â–‘ â–ˆ â–ˆ â–ˆ
         | â–‘ â–‘ â–‘ â–ˆ â–ˆ

    ATTENTION MASK (sum of causal and sliding window masks):

           â–‘ â–ˆ â–ˆ â–‘ â–‘
           â–‘ â–‘ â–ˆ â–ˆ â–‘
           â–‘ â–‘ â–‘ â–ˆ â–ˆ

    r	   .)ÚdtypeÚdevice)ÚdiagonalN)r*   ÚfinforO   r<   ÚrangeÚlenÚsliceÚfullÚshaperP   ÚtriuÚtril)rJ   rK   rL   rM   r@   ÚiÚseqlen_qÚseqlen_kÚcausal_diagonalÚquery_rangeÚ	key_rangeÚ	minus_infÚmaskedÚsliding_diagonals                 r   Úbuild_attention_maskrc   R   sx  € õt ”˜NÔ0Ñ1Ô1Ô5€IÝ•3Ð+Ñ,Ô,¨qÑ0Ñ1Ô1ð =ð =ˆØ'¨¨A©Ô.Ð1EÀaÔ1HÑHˆØ'¨¨A©Ô.Ð1EÀaÔ1HÑHˆØ�hÒÐ 8¨q¢= =Ø&¨Ñ1°AÑ5ˆOˆOàˆOÝÐ0°Ô3Ð5IÈ!ÈaÉ%Ô5PÑQÔQˆÝÐ.¨qÔ1Ð3GÈÈAÉÔ3NÑOÔOˆ	å”JØ˜3 ¨YÐ6Ô7Ô=ØØ Ô&Ø!Ô(ð	
ñ 
ô 
ˆ	õ ”˜I°Ð@Ñ@Ô@ˆà˜AÒÐØ'¨(Ñ2°^ÑCÐØ•e”j Ð5EÐFÑFÔFÑFˆFà6<ˆ�s˜K¨Ð2Ñ3Ð3ð-=ð =r   ÚnumÚstatusÚnum_q_tokensÚmax_kv_readÚcachec           	      ór  ‡— ˆfd„t          | ¦  «        D ¦   «         }||z   }t          ||j        z  ¦  «        }g }|D ]y}	t          |	dg|z  d¬¦  «        }
‰|
_        dg|z  |
_        ||
_        |                     ||
j        d¦  «        }|€|c S | 	                    t          |
dd|¬¦  «        ¦  «         Œz|S )zPA utility function to create a list of FutureRequestStates for the warmup of CB.c                 ó,   •— g | ]}d ‰j         › d|› d�‘ŒS )Ú	__warmup_r   Ú__)Úname)Ú.0rZ   re   s     €r   ú
<listcomp>z/create_warmup_future_states.<locals>.<listcomp>¯   s/   ø€ ÐGÐGÐG°qÐ2˜vœ{Ð2Ð2¨QÐ2Ð2Ð2ÐGÐGÐGr   r   r	   )Ú
request_idÚinitial_tokensÚmax_new_tokensNT)Úhas_new_tokenÚcomplete_blocksÚquery_length)rS   r   Ú
block_sizer   Ú_statusÚtokens_to_processÚposition_offsetÚallocate_blocksrp   Úappendr
   )rd   re   rf   rg   rh   Úrequest_idsÚtotal_tokensÚblocks_neededÚfuture_statesÚreq_idÚstateÚ	allocateds    `          r   Úcreate_warmup_future_statesrƒ   ¦   sø   ø€ ð HÐGÐGÐG½EÀ#¹J¼JÐGÑGÔG€KØ +Ñ-€LÝ˜¨Ô(8Ñ8Ñ9Ô9€Mà€MØð 
ð 
ˆÝ¨À¸sÀ\Ñ?QÐbcÐdÑdÔdˆØˆŒØ#$ #¨Ñ"4ˆÔØ +ˆÔà×)Ò)¨-¸Ô9IÈ1ÑMÔMˆ	ØÐØ Ð Ð Ð Ø×ÒÝ˜u°DÈ!ÐZfÐgÑgÔgñ	
ô 	
ð 	
ð 	
ð Ðr   Úrequest_queuec                 óÞ   — g }|                       ¦   «         sV	 |                      ¦   «         }|�|                     |¦  «         n# t          j        $ r Y nw xY w|                       ¦   «         ¯V|S )z3Drains a queue and returns a list of RequestStates.)ÚemptyÚ
get_nowaitr{   ÚqueueÚEmpty)r„   Ú
new_statesr�   s      r   Údrain_queuer‹   Ã   sŒ   € à%'€JØ×!Ò!Ñ#Ô#ð ð	Ø!×,Ò,Ñ.Ô.ˆEØÐ Ø×!Ò! %Ñ(Ô(Ð(øøÝŒ{ð 	ð 	ð 	ØˆEð	øøøð ×!Ò!Ñ#Ô#ð ð Ðs   ˜+A ÁAÁAc                  óT   — t           j                             ¦   «         } | j        }| |fS )z=Returns a tuple of (mem_pool, graph_pool_id) for CUDA graphs.)r*   r+   ÚMemPoolÚid)Úmem_poolÚgraph_pool_ids     r   Úget_cuda_poolsr‘   Ð   s'   € åŒz×!Ò!Ñ#Ô#€HØ”K€MØ�]Ð"Ð"r   c              #   ó‚   K  — t           j                             | ¦  «        5  dV — ddd¦  «         dS # 1 swxY w Y   dS )z)A context manager to use a CUDA mem pool.N)r*   r+   Úuse_mem_pool)r�   s    r   Úmem_pool_ctxr”   ×   s‹   è è € õ 
Œ×	 Ò	  Ñ	*Ô	*ð ð Øˆˆˆðð ð ñ ô ð ð ð ð ð ð ð øøøð ð ð ð ð ð s   ¢4´8»8)r   )r	   )"rˆ   Ú
contextlibr   Údataclassesr   Úmathr   r   Útypingr   r*   Ú transformers.configuration_utilsr   Úrequestsr
   r   r   r   r/   Úboolr7   r)   r>   rD   rI   ÚTensorÚlistrc   rƒ   ÚQueuer‹   r(   r‘   r”   r-   r   r   ú<module>rŸ      sÍ  ðð €€€Ø %Ð %Ð %Ð %Ð %Ð %Ø !Ð !Ð !Ð !Ð !Ð !Ø Ð Ð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à €€€à =Ð =Ð =Ð =Ð =Ð =à EÐ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EÐ Eð#ð #ð #ð #ð #ñ #ô #ð #ð$ ðð ð ð ð ñ ô ñ „ððHÐ 0ð H°Tð Hð Hð Hð Hð
"˜#ð "¨cð "¸cð "Àcð "ð "ð "ð "ð"ð "�sð " sð "°sð "À3ð "ð "ð "ð "ð�cð  cð °Sð ¸Sð ð ð ð ð ð	Q=ð Q=Ø”LðQ=à˜sœ)ðQ=ð ˜sœ)ðQ=ð ð	Q=ð
 
ðQ=ð Q=ð Q=ð Q=ðhØ	ðàðð ðð ð	ð
 ðð 
Ð
Ôðð ð ð ð:
˜uœ{ð 
¨t°LÔ/Að 
ð 
ð 
ð 
ð#˜ð #ð #ð #ð #ð ðð ñ „ðð ð r   