§
    Ú³¸j—€  ã                   óÆ  — U d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	m	Z	 d dl
mZ d dlmZ d dlmZmZ d dlmZ ddlmZmZmZ dd	lmZmZmZmZ dd
lmZmZ ddlmZm Z m!Z!m"Z" ddl#m$Z$ ddl%m&Z& erddl'm(Z(  ej)        e*¦  «        Z+dZ,dZ-dZ.g d¢Z/e0e1         e2d<   dZ3dZ4dZ5dZ6dZ7dZ8de0e         ddfd„Z9ddddddddœdd d!e:d"e:ez  d#e:d$e:dz  d%e;dz  d&e0e:         e:z  dz  d'e0e:         e:z  dz  d(e1dz  d)e;d*e1fd+„Z< G d,„ d-e j=        ¦  «        Z>e?eef         Z@ G d.„ d/¦  «        ZAd0eAdd d!e:d#e:d$e:f
d1„ZBd0eAde?e>e0e@         f         dz  fd2„ZCd3e@ddfd4„ZDd5e0e@         dd d!e:d#e:d$e:ddfd6„ZEd5e0e@         dd d!e:d#e:d$e:ddfd7„ZFd5e0e@         dd d!e:d#e:d$e:ddfd8„ZG G d9„ d:e¦  «        ZHd3e@deHfd;„ZId<d=de0e@         fd>„ZJd<d=d?e1de0e@         fd@„ZKdAe:ddfdB„ZLdS )Cé    N)Údatetime)ÚPath)ÚLock)ÚTYPE_CHECKINGÚAny)Úquoteé   )ÚCommitOperationAddÚ
UploadInfoÚ_fetch_upload_modes)ÚLocalUploadFileMetadataÚLocalUploadFilePathsÚget_local_upload_pathsÚread_upload_metadata)ÚDEFAULT_REVISIONÚ
REPO_TYPES)ÚDEFAULT_IGNORE_PATTERNSÚ_format_sizeÚfilter_repo_objectsÚtqdm)Úis_xet_available)Úsha_fileobj)ÚHfApié
   é   éd   )
é   é2   éK   r   é}   éÈ   éú   i�  iX  iè  ÚCOMMIT_SIZE_SCALEé   i † i'  r!   r   Ú
paths_listÚreturnc                 óè  — t                                d¦  «         t          | ¦  «        t          k    r5t                                dt          | ¦  «        d›dt          d›d�¦  «         ddlm}  |d„ ¦  «        }| D ]Ä}t          |j        ¦  «        }|j	        }t          |j        ¦  «        d	k    rt          |j        ¦  «        nd	}||         d
xx         dz  cc<   t          |dd…         ¦  «        D ]K\  }}|dk    rd	nd                     |d|…         ¦  «        }||         d                              |¦  «         ŒLŒÅ|                     ¦   «         D ]o\  }	}
|
d
         }t          |
d         ¦  «        }||z   }|t           k    r=|	d	k    rdn|	}t                                d|› d|d›d|d›d|d›dt           d›d�¦  «         Œpg }g }| D ]t}|j                             ¦   «         j        }|dz  }|t(          k    r|                     |j        |f¦  «         ŒM|t,          k    r|                     |j        |f¦  «         Œu|rŠd                     d„ |dd…         D ¦   «         ¦  «        }t          |¦  «        dk    rdt          |¦  «        dz
  › d�nd}t                                dt          |¦  «        › dt(          › d |› |› d!�¦  «         |rŠd                     d"„ |dd…         D ¦   «         ¦  «        }t          |¦  «        dk    rdt          |¦  «        dz
  › d�nd}t                                dt          |¦  «        › d#t,          › d$|› |› d%�¦  «         t                                d&¦  «         dS )'as  
    Validate upload against repository limits and warn about potential issues.

    Args:
        paths_list: List of file paths to be uploaded

    Warns about:
        - Too many files in the repository (>100k)
        - Too many entries (files or subdirectories) in a single folder (>10k)
        - Files exceeding size limits (>20GB recommended, >200GB maximum)
    z/Running validation checks on files to upload...zYou are about to upload ú,z. files. This exceeds the recommended limit of zÍ files per repository.
Consider:
  - Splitting your data into multiple repositories
  - Using fewer, larger files (e.g., parquet files)
  - See: https://huggingface.co/docs/hub/repositories-recommendationsr   )Údefaultdictc                  ó$   — dt          ¦   «         dœS )Nr   )ÚfilesÚsubdirs)Úset© ó    út/root/private/project/pallet_detection_v2/.venv/lib/python3.11/site-packages/huggingface_hub/_upload_large_folder.pyú<lambda>z)_validate_upload_limits.<locals>.<lambda>V   s   € ÀqÕUXÑUZÔUZÐ=[Ð=[€ r/   ú.r+   r	   Néÿÿÿÿú/r,   ÚrootzFolder 'z' contains z
 entries (z files and z/ subdirectories). This exceeds the recommended z< entries per folder.
Consider reorganising into sub-folders.i Êš;z
  - c              3   ó.   K  — | ]\  }}|› d |d›d�V — ŒdS ©z: z.1fÚGBNr.   ©Ú.0ÚpathÚsizes      r0   ú	<genexpr>z*_validate_upload_limits.<locals>.<genexpr>„   s:   è è € Ð!aÐ!a¹j¸dÀD TÐ"9Ð"9¨TÐ"9Ð"9Ð"9Ð"9Ð!aÐ!aÐ!aÐ!aÐ!aÐ!ar/   é   z
  ... and z more filesÚ úFound z files exceeding the zGB recommended maximum:
  - z4
Consider splitting these files into smaller chunks.c              3   ó.   K  — | ]\  }}|› d |d›d�V — ŒdS r7   r.   r9   s      r0   r=   z*_validate_upload_limits.<locals>.<genexpr>Ž   s:   è è € Ð!\Ð!\¹j¸dÀD TÐ"9Ð"9¨TÐ"9Ð"9Ð"9Ð"9Ð!\Ð!\Ð!\Ð!\Ð!\Ð!\r/   z files larger than zGB (recommended limit):
  - z2
Large files may slow down loading and processing.zValidation checks complete.)ÚloggerÚinfoÚlenÚMAX_FILES_PER_REPOÚwarningÚcollectionsr)   r   Úpath_in_repoÚpartsÚstrÚparentÚ	enumerateÚjoinÚaddÚitemsÚMAX_FILES_PER_FOLDERÚ	file_pathÚstatÚst_sizeÚMAX_FILE_SIZE_GBÚappendÚRECOMMENDED_FILE_SIZE_GB)r%   r)   Úentries_per_folderÚpathsr;   rI   rK   ÚiÚchildÚfolderÚdataÚ
file_countÚsubdir_countÚtotal_entriesÚfolder_displayÚlarge_filesÚvery_large_filesr<   Úsize_gbÚ	files_strÚmore_strs                        r0   Ú_validate_upload_limitsrf   9   s“  € õ ‡K‚KÐAÑBÔBÐBõ ˆ:�„Õ+Ò+Ð+Ý�ŠðU¥s¨:¡¤ÐBð Uð UÝ5GÐcðUð Uð Uñ	
ô 	
ð 	
ð (Ð'Ð'Ð'Ð'Ð'à)4¨Ð5[Ð5[Ñ)\Ô)\Ðàð =ð =ˆÝ�EÔ&Ñ'Ô'ˆØ”
ˆõ &)¨¬Ñ%5Ô%5¸Ò%<Ð%<•�T”[Ñ!Ô!Ð!À#ˆØ˜6Ô" 7Ð+Ð+Ô+¨qÑ0Ð+Ð+Ñ+õ " %¨¨¨¤*Ñ-Ô-ð 	=ð 	=‰HˆAˆuØ šF˜F�S�S¨¯ª°°r¸°r´Ñ(;Ô(;ˆFØ˜vÔ& yÔ1×5Ò5°eÑ<Ô<Ð<Ð<ð	=ð
 +×0Ò0Ñ2Ô2ð ð ‰ˆ�Ø˜'”]ˆ
Ý˜4 	œ?Ñ+Ô+ˆØ" \Ñ1ˆàÕ/Ò/Ð/Ø'-°¢} }˜V˜V¸&ˆNÝ�NŠNð:˜>ð :ð :°mÐPð :ð :ØÐOð:ð :Ø-9ÐOð:ð :å0DÐ^ð:ð :ð :ñô ð øð €KØÐàð >ð >ˆØŒ×#Ò#Ñ%Ô%Ô-ˆØ˜Ñ&ˆàÕ%Ò%Ð%Ø×#Ò# UÔ%7¸Ð$AÑBÔBÐBÐBØÕ/Ò/Ð/Ø×Ò Ô 2°GÐ<Ñ=Ô=Ð=øð ð 
Ø—M’MÐ!aÐ!aÐL\Ð]_Ð^_Ð]_ÔL`Ð!aÑ!aÔ!aÑaÔaˆ	ÝLOÐP`ÑLaÔLaÐdeÒLeÐLeÐH¥#Ð&6Ñ"7Ô"7¸!Ñ";ÐHÐHÐHÐHÐkmˆÝ�ŠðC•SÐ)Ñ*Ô*ð Cð CÕAQð Cð CØðCØ&ðCð Cð Cñ	
ô 	
ð 	
ð ð 
Ø—M’MÐ!\Ð!\ÈKÐXZÐYZÐXZÌOÐ!\Ñ!\Ô!\Ñ\Ô\ˆ	ÝGJÈ;ÑGWÔGWÐZ[ÒG[ÐG[ÐC¥# kÑ"2Ô"2°QÑ"6ÐCÐCÐCÐCÐacˆÝ�ŠðA•S˜Ñ%Ô%ð Að AÕ:Rð Að AØðAØ&ðAð Að Añ	
ô 	
ð 	
õ ‡K‚KÐ-Ñ.Ô.Ð.Ð.Ð.r/   Té<   )ÚrevisionÚprivateÚallow_patternsÚignore_patternsÚnum_workersÚprint_reportÚprint_report_everyÚapir   Úrepo_idÚfolder_pathÚ	repo_typerh   ri   rj   rk   rl   rm   rn   c                ó’  ‡ ‡‡‡‡‡— ‰€t          d¦  «        ‚‰t          vrt          dt          › �¦  «        ‚‰€t          Št          ‰¦  «                             ¦   «                              ¦   «         Š‰                     ¦   «         st          d‰› d�¦  «        ‚|€g }nt          |t          ¦  «        r|g}|t          z  }|€(t          j        ¦   «         pd}t          |dz  d¦  «        }‰                      ‰‰|d¬	¦  «        }t                               d
|› �¦  «         |j        Š	 ‰                      ‰‰‰¬¦  «        }t%          |¦  «        }|dk    r*t                               dd› d|› d‰› d‰› dd› d�¦  «         n4# t(          $ r'}t                               d|› �¦  «         Y d}~nd}~ww xY wt-          ¦   «         rt.          nt0          }t3          ˆfd„‰                     d¦  «        D ¦   «         ||¬¦  «        }ˆfd„|D ¦   «         }t                               dt%          |¦  «        › d�¦  «         t7          |¦  «         t                               d¦  «         ˆfd„t9          |d¬¦  «        D ¦   «         }t;          ||¦  «        Šˆ ˆˆˆˆfd„t=          |¦  «        D ¦   «         }|D ]}|                     ¦   «          Œ|	r$tA          d‰ !                    ¦   «         z   ¦  «         tE          j"        ¦   «         }	 tE          j#        d¦  «         tE          j"        ¦   «         |z
  |
k    r6|	r!tI          ‰ !                    ¦   «         ¦  «         tE          j"        ¦   «         }‰j%        �$t           &                    d tN          › d!�¦  «         n0‰ (                    ¦   «         rt                               d"¦  «         nŒ¿|D ]}| )                    ¦   «          Œ‰j%        �tU          d#tN          › d$�¦  «        ‰j%        ‚t                               ‰ !                    ¦   «         ¦  «         t                               d%¦  «         dS )&zŒUpload a large folder to the Hub in the most resilient way possible.

    See [`HfApi.upload_large_folder`] for the full documentation.
    Nz For large uploads, `repo_type` is explicitly required. Please set it to `model`, `dataset` or `space`. If you are using the CLI, pass it as `--repo-type=model`.z"Invalid repo type, must be one of zProvided path: 'z' is not a directoryr	   é   T)rp   rr   ri   Úexist_okzRepo created: )rp   rr   rh   iô  ú
zP================================================================================z
WARNING: This repository has aR   commits.
Repositories with a large number of commits can experience performance issues.

Consider squashing your commit history using `super_squash_history()`.
To do so, you need to stop this process, run the snippet below and restart the upload command.  from huggingface_hub import super_squash_history
  super_squash_history(repo_id='z', repo_type='zÎ')

Note: This is a non-revertible operation. See the documentation for more details:
https://huggingface.co/docs/huggingface_hub/main/en/package_reference/hf_api#huggingface_hub.HfApi.super_squash_history
zCould not check commit count: c              3   óŽ   •K  — | ]?}|                      ¦   «         ¯|                     ‰¦  «                             ¦   «         V — Œ@d S ©N)Úis_fileÚrelative_toÚas_posix)r:   r;   rq   s     €r0   r=   z/upload_large_folder_internal.<locals>.<genexpr>å   sO   øè è € ÐiÐi°dÐZ^×ZfÒZfÑZhÔZhÐiˆ×	Ò	˜+Ñ	&Ô	&×	/Ò	/Ñ	1Ô	1ÐiÐiÐiÐiÐiÐir/   z**/*)rj   rk   c                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r.   )r   )r:   Úrelpathrq   s     €r0   ú
<listcomp>z0upload_large_folder_internal.<locals>.<listcomp>é   s$   ø€ ÐbÐbÐbÀ7Õ(¨°gÑ>Ô>ÐbÐbÐbr/   r@   z candidate files to uploadzStarting upload...c                 ó>   •— g | ]}|t          ‰|j        ¦  «        f‘ŒS r.   )r   rH   )r:   rX   rq   s     €r0   r~   z0upload_large_folder_internal.<locals>.<listcomp>ò   s<   ø€ ð ð ð àð 
Õ$ [°%Ô2DÑEÔEÐFðð ð r/   zRecovering from metadata files)Údescc                 óR   •— g | ]#}t          j        t          ‰‰‰‰‰d œ¬¦  «        ‘Œ$S ))Ústatusro   rp   rr   rh   )ÚtargetÚkwargs)Ú	threadingÚThreadÚ_worker_job)r:   Ú_ro   rp   rr   rh   r‚   s     €€€€€r0   r~   z0upload_large_folder_internal.<locals>.<listcomp>ù   sY   ø€ ð ð ð ð õ 	ÔÝà ØØ"Ø&Ø$ðð ð		
ñ 		
ô 		
ðð ð r/   z

zToo many consecutive errors (z): exiting main loopzIs done: exiting main loopzUpload aborted after zR consecutive errors. Progress is saved locally: re-run the same command to resume.zUpload is complete!)+Ú
ValueErrorr   r   r   Ú
expanduserÚresolveÚis_dirÚ
isinstancerJ   r   ÚosÚ	cpu_countÚmaxÚcreate_reporB   rC   rp   Úlist_repo_commitsrD   rF   Ú	ExceptionÚdebugr   ÚUPLOAD_BATCH_SIZE_XETÚUPLOAD_BATCH_SIZE_LFSr   Úglobrf   r   ÚLargeUploadStatusÚrangeÚstartÚprintÚcurrent_reportÚtimeÚsleepÚ_print_overwriteÚfatal_errorÚerrorÚMAX_CONSECUTIVE_ERRORSÚis_donerM   ÚRuntimeError)ro   rp   rq   rr   rh   ri   rj   rk   rl   rm   rn   Únb_coresÚrepo_urlÚcommitsÚcommit_countÚeÚupload_batch_sizeÚfiltered_paths_listr%   rO   ÚthreadsÚthreadÚlast_report_tsr‚   s   `````                  @r0   Úupload_large_folder_internalr¯   ™   s  øøøøøø€ ð& ÐÝðIñ
ô 
ð 	
ð �
Ð"Ð"ÝÐJ½jÐJÐJÑKÔKÐKØÐÝ#ˆå�{Ñ#Ô#×.Ò.Ñ0Ô0×8Ò8Ñ:Ô:€KØ×ÒÑÔð OÝÐM¨KÐMÐMÐMÑNÔNÐNàÐØˆˆÝ	�O¥SÑ	)Ô	)ð ,Ø*Ð+ˆØÕ.Ñ.€OàÐÝ”<‘>”>Ð& QˆÝ˜( a™-¨Ñ+Ô+ˆð �Š w¸)ÈWÐ_cˆÑdÔd€HÝ
‡K‚KÐ+ Ð+Ð+Ñ,Ô,Ð,ØÔ€Gð;Ø×'Ò'°À9ÐW_Ð'Ñ`Ô`ˆÝ˜7‘|”|ˆØ˜#ÒÐÝ�NŠNð �Xð  ð  Ø0<ð ð  ð 4;ð ð  ð KTð ð  ð ð ð  ð  ñô ð øøõ ð ;ð ;ð ;å�ŠÐ9°aÐ9Ð9Ñ:Ô:Ð:Ð:Ð:Ð:Ð:Ð:øøøøð;øøøõ
 2BÑ1CÔ1CÐ^Õ-Ð-ÕI^Ðõ .ØiÐiÐiÐi¸k×>NÒ>NÈvÑ>VÔ>VÐiÑiÔiØ%Ø'ðñ ô Ðð
 cÐbÐbÐbÐNaÐbÑbÔb€JÝ
‡K‚KÐD�˜Z™œÐDÐDÐDÑEÔEÐEõ ˜JÑ'Ô'Ð'å
‡K‚KÐ$Ñ%Ô%Ð%ðð ð ð å˜*Ð+KÐLÑLÔLðñ ô €Eõ ˜uÐ&7Ñ8Ô8€Fðð ð ð ð ð ð ð õ �{Ñ#Ô#ðñ ô €Gð ð ð ˆØ�Š‰Œˆˆð ð 0Ýˆf�v×,Ò,Ñ.Ô.Ñ.Ñ/Ô/Ð/Ý”Y‘[”[€NðÝŒ
�1‰ŒˆÝŒ9‰;Œ;˜Ñ'Ð+=Ò=Ð=Øð :Ý  ×!6Ò!6Ñ!8Ô!8Ñ9Ô9Ð9Ý!œY™[œ[ˆNØÔÐ)Ý�LŠLÐeÕ9OÐeÐeÐeÑfÔfÐfØØ�>Š>ÑÔð 	Ý�KŠKÐ4Ñ5Ô5Ð5Øðð ð ð ˆØ�Š‰ŒˆˆàÔÐ%Ýð'Õ$:ð 'ð 'ð 'ñ
ô 
ð Ô!ð	"õ
 ‡K‚K�×%Ò%Ñ'Ô'Ñ(Ô(Ð(Ý
‡K‚KÐ%Ñ&Ô&Ð&Ð&Ð&s   Ä*AF Æ
F3ÆF.Æ.F3c                   ó¤   — e Zd Z ej        ¦   «         Z ej        ¦   «         Z ej        ¦   «         Z ej        ¦   «         Z ej        ¦   «         Z	dS )Ú	WorkerJobN)
Ú__name__Ú
__module__Ú__qualname__ÚenumÚautoÚSHA256ÚGET_UPLOAD_MODEÚPREUPLOAD_LFSÚCOMMITÚWAITr.   r/   r0   r±   r±   -  sO   € € € € € ØˆTŒY‰[Œ[€FØ�d”i‘k”k€OØ�D”I‘K”K€MØˆTŒY‰[Œ[€FØˆ4Œ9‰;Œ;€D€D€Dr/   r±   c                   ó„   — e Zd ZdZddee         defd„Zdd„Zd	e	ddfd
„Z
defd„Zdedededdfd„Zdefd„Zdefd„ZdS )r˜   zBContains information, queues and tasks for a large upload process.r	   rO   rª   c                 óf  — || _         t          j        ¦   «         | _        t          j        ¦   «         | _        t          j        ¦   «         | _        t          j        ¦   «         | _        t          ¦   «         | _        d| _	        d| _
        d| _        || _        d| _        d| _        d | _        d| _        d | _        t%          j        ¦   «         | _        d| _        t          ¦   «         | _        | j         D ]½}|\  }}|j        €| j                             |¦  «         Œ)|j        €| j                             |¦  «         ŒK|j        dk    r"|j        s| j                             |¦  «         Œx|j        s| j                             |¦  «         Œšt8                               d|j        › d�¦  «         Œ¾d S )Nr   r	   ÚlfszSkipping file z! (already uploaded and committed))rO   ÚqueueÚQueueÚqueue_sha256Úqueue_get_upload_modeÚqueue_preupload_lfsÚqueue_commitr   ÚlockÚnb_workers_sha256Únb_workers_get_upload_modeÚnb_workers_preupload_lfsrª   Únb_workers_commitÚnb_workers_waitingÚlast_commit_attemptÚnb_consecutive_errorsr    r   ÚnowÚ_started_atÚ
_chunk_idxÚ_chunk_lockÚsha256ÚputÚupload_modeÚis_uploadedÚis_committedrB   r”   rH   )ÚselfrO   rª   ÚitemrX   Úmetadatas         r0   Ú__init__zLargeUploadStatus.__init__;  sž  € ØˆŒ
Ý7<´{±}´}ˆÔÝ@EÄÁÄˆÔ"Ý>C¼k¹m¼mˆÔ Ý7<´{±}´}ˆÔÝ‘F”FˆŒ	à&'ˆÔØ/0ˆÔ'Ø-.ˆÔ%Ø&7ˆÔØ&'ˆÔØ'(ˆÔØ15ˆÔ Ø*+ˆÔ"Ø-1ˆÔå#œ<™>œ>ˆÔØ ˆŒÝ!%¡¤ˆÔð ”Jð 	eð 	eˆDØ"‰OˆE�8ØŒÐ&ØÔ!×%Ò% dÑ+Ô+Ð+Ð+ØÔ%Ð-ØÔ*×.Ò.¨tÑ4Ô4Ð4Ð4ØÔ%¨Ò.Ð.°xÔ7KÐ.ØÔ(×,Ò,¨TÑ2Ô2Ð2Ð2ØÔ*ð eØÔ!×%Ò% dÑ+Ô+Ð+Ð+å—’Ðc¨eÔ.@ÐcÐcÐcÑdÔdÐdÐdð	eð 	er/   r&   Nc                 óT   — | j         5  d| _        d d d ¦  «         d S # 1 swxY w Y   d S )Nr   )rÅ   rÌ   ©rÖ   s    r0   Úreport_successz LargeUploadStatus.report_success_  sv   € ØŒYð 	+ð 	+Ø)*ˆDÔ&ð	+ð 	+ð 	+ñ 	+ô 	+ð 	+ð 	+ð 	+ð 	+ð 	+ð 	+ð 	+øøøð 	+ð 	+ð 	+ð 	+ð 	+ð 	+s   ˆ�!¤!Úexcc                 ó”   — | j         5  | xj        dz  c_        | j        t          k    r|| _        ddd¦  «         dS # 1 swxY w Y   dS )a-  Give up on the whole upload once errors keep piling up.

        Failed items are put back in their queue and retried indefinitely, which spins forever (and hammers the Hub)
        when the error is permanent. Stop after `MAX_CONSECUTIVE_ERRORS` failures without a single success in between.
        r	   N)rÅ   rÌ   r¢   r    )rÖ   rÝ   s     r0   Úreport_errorzLargeUploadStatus.report_errorc  s›   € ð ŒYð 	'ð 	'ØÐ&Ô&¨!Ñ+Ð&Ô&ØÔ)Õ-CÒCÐCØ#&�Ô ð	'ð 	'ð 	'ñ 	'ô 	'ð 	'ð 	'ð 	'ð 	'ð 	'ð 	'ð 	'øøøð 	'ð 	'ð 	'ð 	'ð 	'ð 	's   ˆ(=½AÁAc                 óh   — | j         5  t          | j                 cd d d ¦  «         S # 1 swxY w Y   d S rx   )rÐ   r#   rÏ   rÛ   s    r0   Útarget_chunkzLargeUploadStatus.target_chunkn  sy   € ØÔð 	6ð 	6Ý$ T¤_Ô5ð	6ð 	6ð 	6ð 	6ñ 	6ô 	6ð 	6ð 	6ð 	6ð 	6ð 	6ð 	6øøøð 	6ð 	6ð 	6ð 	6ð 	6ð 	6s   ˆ'§+®+ÚsuccessÚnb_itemsÚdurationc           
      ó¶  — | j         5  |s/t                               d|› d�¦  «         | xj        dz  c_        nJ|t          | j                 k    r4|dk     r.t                               d|› d�¦  «         | xj        dz  c_        t          dt          | j        t          t          ¦  «        dz
  ¦  «        ¦  «        | _        d d d ¦  «         d S # 1 swxY w Y   d S )NzFailed to commit z9 files at once. Will retry with less files in next batch.r	   é(   zSuccessfully committed z. at once. Increasing the limit for next batch.r   )	rÐ   rB   rF   rÏ   r#   rC   r�   ÚminrD   )rÖ   râ   rã   rä   s       r0   Úupdate_chunkzLargeUploadStatus.update_chunkr  s0  € ØÔð 	Wð 	WØð %Ý—’Ðv°8ÐvÐvÐvÑwÔwÐwØ�” 1Ñ$�”�ØÕ.¨t¬Ô?Ò?Ð?ÀHÈrÂMÀMÝ—’Ðn°hÐnÐnÐnÑoÔoÐoØ�” 1Ñ$�”å! !¥S¨¬½#Õ>OÑ:PÔ:PÐSTÑ:TÑ%UÔ%UÑVÔVˆDŒOð	Wð 	Wð 	Wñ 	Wô 	Wð 	Wð 	Wð 	Wð 	Wð 	Wð 	Wð 	Wøøøð 	Wð 	Wð 	Wð 	Wð 	Wð 	Ws   ˆB9CÃCÃCc                 ó  — d}d}d}d}d}d}d}d}d}	d}
d}| j         5  | j        D ]\  }}|j        r|
dz  }
Œ|	|j        z  }	|dz  }|j        �|dz  }||j        z  }|j        dk    r|dz  }|j        €|dz  }|j        r|dz  }||j        z  }|j        r|dz  }||j        z  }Œ€t          |	¦  «        }t          j
        ¦   «         }|                     d¦  «        }|| j        z
  }t          |¦  «                             d¦  «        d         }d}|d|› d	|› d
�z  }|dz  }|dz  }|d|› d|› d	t          |¦  «        › d|› d�	z  }|d|› d|› d	t          |¦  «        › d|› d�	z  }|dk    r	|d|› d�z  }|d|› d|› d	t          |¦  «        › d|› d�	z  }|d|
› d�z  }|dz  }|d| j        › d�z  }|d| j        › d�z  }|d| j        › d�z  }|d| j        › d�z  }|d| j        › d�z  }|dz  }|cddd¦  «         S # 1 swxY w Y   dS )z<Generate a report of the current status of the large upload.r   r	   Nr¾   z%Y-%m-%d %H:%M:%Sr2   z
----------ú z (z) z----------
z	Files:   zhashed r4   z) | zpre-uploaded: ú)z (+z unsure)z | committed: z | ignored: rv   z	Workers: z	hashing: z | zget upload mode: zpre-uploading: zcommitting: z	waiting: z3---------------------------------------------------)rÅ   rO   Úshould_ignorer<   rÑ   rÓ   rÔ   rÕ   r   r   rÍ   ÚstrftimerÎ   rJ   ÚsplitrÆ   rÇ   rÈ   rÉ   rÊ   )rÖ   Ú	nb_hashedÚsize_hashedÚnb_preuploadedÚnb_lfsÚnb_lfs_unsureÚsize_preuploadedÚnb_committedÚsize_committedÚ
total_sizeÚignored_filesÚtotal_filesrˆ   rØ   Útotal_size_strrÍ   Únow_strÚelapsedÚelapsed_strÚmessages                       r0   rœ   z LargeUploadStatus.current_report}  ss  € àˆ	ØˆØˆØˆØˆØÐØˆØˆØˆ
ØˆØˆàŒYð /	ð /	Ø#œzð 4ð 4‘��8ØÔ)ð Ø! QÑ&�MØØ˜hœmÑ+�
Ø˜qÑ �Ø”?Ð.Ø ‘N�IØ 8¤=Ñ0�KØÔ'¨5Ò0Ð0Ø˜a‘K�FØÔ'Ð/Ø! QÑ&�MØÔ'ð 6Ø" aÑ'�NØ$¨¬Ñ5Ð$ØÔ(ð 4Ø  AÑ%�LØ" h¤mÑ3�NøÝ)¨*Ñ5Ô5ˆNå”,‘.”.ˆCØ—l’lÐ#6Ñ7Ô7ˆGØ˜DÔ,Ñ,ˆGÝ˜g™,œ,×,Ò,¨SÑ1Ô1°!Ô4ˆKà%ˆGØÐ5˜7Ð5Ð5 kÐ5Ð5Ð5Ñ5ˆGØ�Ñ&ˆGà�{Ñ"ˆGØÐl ÐlÐl¨[ÐlÐl½LÈÑ<UÔ<UÐlÐlÐXfÐlÐlÐlÑlˆGØÐu¨ÐuÐu¸ÐuÐuÅ<ÐP`ÑCaÔCaÐuÐuÐdrÐuÐuÐuÑuˆGØ˜qÒ Ð ØÐ8 Ð8Ð8Ð8Ñ8�ØÐv¨ÐvÐv°{ÐvÐvÅlÐSaÑFbÔFbÐvÐvÐesÐvÐvÐvÑvˆGØÐ7 mÐ7Ð7Ð7Ñ7ˆGà�{Ñ"ˆGØÐ> 4Ô#9Ð>Ð>Ð>Ñ>ˆGØÐO¨4Ô+JÐOÐOÐOÑOˆGØÐK¨Ô)FÐKÐKÐKÑKˆGØÐA dÔ&<ÐAÐAÐAÑAˆGØÐ> 4Ô#:Ð>Ð>Ð>Ñ>ˆGØ�xÑˆGàð_/	ð /	ð /	ð /	ñ /	ô /	ð /	ð /	ð /	ð /	ð /	ð /	øøøð /	ð /	ð /	ð /	ð /	ð /	s   žGG:Ç:G>ÈG>c                 ó€   — | j         5  t          d„ | j        D ¦   «         ¦  «        cd d d ¦  «         S # 1 swxY w Y   d S )Nc              3   ó8   K  — | ]\  }}|j         p|j        V — Œd S rx   ©rÕ   rì   ©r:   rˆ   rØ   s      r0   r=   z,LargeUploadStatus.is_done.<locals>.<genexpr>¾  s1   è è € ÐeÐeÉ;È1Èh�xÔ,ÐF°Ô0FÐeÐeÐeÐeÐeÐer/   )rÅ   ÚallrO   rÛ   s    r0   r£   zLargeUploadStatus.is_done¼  s    € ØŒYð 	fð 	fÝÐeÐeÐZ^ÔZdÐeÑeÔeÑeÔeð	fð 	fð 	fð 	fñ 	fô 	fð 	fð 	fð 	fð 	fð 	fð 	føøøð 	fð 	fð 	fð 	fð 	fð 	fs   ˆ3³7º7)r	   ©r&   N)r²   r³   r´   Ú__doc__ÚlistÚ
JOB_ITEM_TÚintrÙ   rÜ   r“   rß   rá   ÚboolÚfloatrè   rJ   rœ   r£   r.   r/   r0   r˜   r˜   8  s  € € € € € ØLÐLð"eð "e˜d :Ô.ð "eÀ3ð "eð "eð "eð "eðH+ð +ð +ð +ð	' 	ð 	'¨dð 	'ð 	'ð 	'ð 	'ð6˜cð 6ð 6ð 6ð 6ð	W Dð 	W°Cð 	WÀ5ð 	WÈTð 	Wð 	Wð 	Wð 	Wð= ð =ð =ð =ð =ð~f˜ð fð fð fð fð fð fr/   r˜   r‚   c                 óv
  — 	 | j         �dS d}t          | ¦  «        }|€dS |\  }}|xt          j        k    rù |d         }	 t	          |¦  «         | j                             |¦  «         |                      ¦   «          n€# t          $ r ‚ t          $ ri}	t                               d|	› �¦  «         t          j        ¦   «          | j                             |¦  «         |                      |	¦  «         Y d}	~	nd}	~	ww xY w| j        5  | xj        dz  c_        ddd¦  «         n# 1 swxY w Y   �n
xt          j        k    �r7 	 t'          |||||¬¦  «         |                      ¦   «          nf# t          $ r ‚ t          $ rO}	t                               d|	› �¦  «         t          j        ¦   «          |                      |	¦  «         Y d}	~	nd}	~	ww xY w|D ]r}|\  }
}|j        rŒ|j        xdk    r | j                             |¦  «         Œ7d	k    r| j                             |¦  «         ŒW	 | j                             |¦  «         Œs| j        5  | xj        dz  c_        ddd¦  «         n# 1 swxY w Y   �nÂxt          j        k    �r  	 t5          |||||¬¦  «         |D ]}| j                             |¦  «         Œ|                      ¦   «          n…# t          $ r ‚ t          $ rn}	t                               d
|	› �¦  «         t          j        ¦   «          |D ]}| j                             |¦  «         Œ|                      |	¦  «         Y d}	~	nd}	~	ww xY w| j        5  | xj        dz  c_        ddd¦  «         n# 1 swxY w Y   �n±xt          j        k    �rI t;          j        ¦   «         }d}	 t=          |||||¬¦  «         |                      ¦   «          n‡# t          $ r ‚ t          $ rp}	t                               d|	› �¦  «         t          j        ¦   «          |D ]}| j                             |¦  «         Œd}|                      |	¦  «         Y d}	~	nd}	~	ww xY wt;          j        ¦   «         |z
  }|                      |tA          |¦  «        |¦  «         | j        5  t;          j        ¦   «         | _!        | xj"        dz  c_"        ddd¦  «         n# 1 swxY w Y   nWt          j#        k    rHt;          j$        tJ          ¦  «         | j        5  | xj&        dz  c_&        ddd¦  «         n# 1 swxY w Y   �Œ9)aª  
    Main process for a worker. The worker will perform tasks based on the priority list until all files are uploaded
    and committed. If no tasks are available, the worker will wait for 10 seconds before checking again.

    If a task fails for any reason, the item(s) are put back in the queue for another worker to pick up.

    Read `upload_large_folder` docstring for more information on how tasks are prioritized.
    TNr   zFailed to compute sha256: r	   )ro   rp   rr   rh   zFailed to get upload mode: r¾   ÚregularzFailed to preupload LFS: zFailed to commit: F)'r    Ú_determine_next_jobr±   r·   Ú_compute_sha256rÂ   rÒ   rÜ   ÚKeyboardInterruptr“   rB   r¡   Ú	tracebackÚ
format_excrÁ   rß   rÅ   rÆ   r¸   Ú_get_upload_moderì   rÓ   rÃ   rÄ   rÇ   r¹   Ú_preupload_lfsrÈ   rº   r�   Ú_commitrè   rD   rË   rÉ   r»   rž   ÚWAITING_TIME_IF_NO_TASKSrÊ   )r‚   ro   rp   rr   rh   Únext_jobÚjobrO   r×   r©   rˆ   rØ   Ústart_tsrâ   rä   s                  r0   r‡   r‡   Á  sN  € ðg3ØÔÐ)ØˆFØ>Bˆõ ' vÑ.Ô.ˆØÐØˆFØ‰
ˆˆUð Ø!•Ô!Ò!Ð!Ð!Ø˜Q”x�ð
+Ý# DÑ)Ô)Ð)ØÔ0×4Ò4°TÑ:Ô:Ð:Ø×)Ò)Ñ+Ô+Ð+Ð+øÝ(ð ð ð ØÝ ð +ð +ð +Ý—L’LÐ!A¸aÐ!AÐ!AÑBÔBÐBÝÔ(Ñ*Ô*Ð*ØÔ'×+Ò+¨DÑ1Ô1Ð1Ø×'Ò'¨Ñ*Ô*Ð*Ð*Ð*Ð*Ð*Ð*øøøøð	+øøøð ”[ð 2ð 2ØÐ,Ô,°Ñ1Ð,Ô,ð2ð 2ð 2ñ 2ô 2ð 2ð 2ð 2ð 2ð 2ð 2øøøð 2ð 2ð 2ð 2ùð +•Ô*Ò*Ñ*Ð*ð+Ý$ U°¸WÐPYÐdlÐmÑmÔmÐmØ×)Ò)Ñ+Ô+Ð+Ð+øÝ(ð ð ð ØÝ ð +ð +ð +Ý—L’LÐ!B¸qÐ!BÐ!BÑCÔCÐCÝÔ(Ñ*Ô*Ð*Ø×'Ò'¨Ñ*Ô*Ð*Ð*Ð*Ð*Ð*Ð*øøøøð+øøøð "ð 
Cð 
C�DØ"&‘K�A�xØÔ-ð !Ø Ø"Ô.Ø"˜UšU˜U˜UØ"Ô6×:Ò:¸4Ñ@Ô@Ð@Ð@Ø&šY˜YØ"Ô/×3Ò3°DÑ9Ô9Ð9Ð9ØØ"Ô8×<Ò<¸TÑBÔBÐBÐBà”[ð ;ð ;ØÐ5Ô5¸Ñ:Ð5Ô5ð;ð ;ð ;ñ ;ô ;ð ;ð ;ð ;ð ;ð ;ð ;øøøð ;ð ;ð ;ð ;ùð )•Ô(Ò(Ñ(Ð(ð+Ý" 5¨c¸7ÈiÐbjÐkÑkÔkÐkØ %ð 6ð 6˜ØÔ+×/Ò/°Ñ5Ô5Ð5Ð5Ø×)Ò)Ñ+Ô+Ð+Ð+øÝ(ð ð ð ØÝ ð +ð +ð +Ý—L’LÐ!@¸QÐ!@Ð!@ÑAÔAÐAÝÔ(Ñ*Ô*Ð*Ø %ð =ð =˜ØÔ2×6Ò6°tÑ<Ô<Ð<Ð<Ø×'Ò'¨Ñ*Ô*Ð*Ð*Ð*Ð*Ð*Ð*øøøøð+øøøð ”[ð 9ð 9ØÐ3Ô3°qÑ8Ð3Ô3ð9ð 9ð 9ñ 9ô 9ð 9ð 9ð 9ð 9ð 9ð 9øøøð 9ð 9ð 9ð 9ùð "•Ô!Ò!Ñ!Ð!Ýœ9™;œ;�Ø�ð+Ý˜E s°GÀyÐ[cÐdÑdÔdÐdØ×)Ò)Ñ+Ô+Ð+Ð+øÝ(ð ð ð ØÝ ð +ð +ð +Ý—L’LÐ!9°aÐ!9Ð!9Ñ:Ô:Ð:ÝÔ(Ñ*Ô*Ð*Ø %ð 6ð 6˜ØÔ+×/Ò/°Ñ5Ô5Ð5Ð5Ø#�GØ×'Ò'¨Ñ*Ô*Ð*Ð*Ð*Ð*Ð*Ð*øøøøð+øøøõ  œ9™;œ;¨Ñ1�Ø×#Ò# G­S°©Z¬Z¸ÑBÔBÐBØ”[ð 2ð 2Ý15´±´�FÔ.ØÐ,Ô,°Ñ1Ð,Ô,ð2ð 2ð 2ñ 2ô 2ð 2ð 2ð 2ð 2ð 2ð 2øøøð 2ð 2ð 2ð 2øõ ”’�Ý”
Õ3Ñ4Ô4Ð4Ø”[ð 3ð 3ØÐ-Ô-°Ñ2Ð-Ô-ð3ð 3ð 3ñ 3ô 3ð 3ð 3ð 3ð 3ð 3ð 3øøøð 3ð 3ð 3ð 3ñMg3s¿   Á =A> Á>C;ÂAC6Ã6C;ÄD"Ä"D&Ä)D&Å(E+ Å+GÅ?AG	Ç	GÉI*É*I.É1I.Ê
AK ËMË&A$MÍMÍM;Í;M?ÎM?Î0(O ÏQÏ-A&QÑQÒ!)SÓSÓSÔT.Ô.T2Ô5T2c                 óL  — | j         5  | j        dk    r¬| j                             ¦   «         dk    r�| j        �ˆt          j        ¦   «         | j        z
  dk    ri| xj        dz  c_        t                               d¦  «         t          j	        t          | j        |                      ¦   «         ¦  «        fcd d d ¦  «         S | j        dk    r†| j                             ¦   «         dk    ri| xj        dz  c_        t                               d¦  «         t          j	        t          | j        |                      ¦   «         ¦  «        fcd d d ¦  «         S | j                             ¦   «         t          k    re| xj        dz  c_        t                               dt          › d�¦  «         t          j        t          | j        t          ¦  «        fcd d d ¦  «         S | j                             ¦   «         | j        k    rg| j        dk    r\| xj        dz  c_        t                               d	¦  «         t          j        t          | j        | j        ¦  «        fcd d d ¦  «         S | j                             ¦   «         dk    ra| j        dk    rV| xj        dz  c_        t                               d
¦  «         t          j        t/          | j        ¦  «        fcd d d ¦  «         S | j                             ¦   «         dk    rg| j        dk    r\| xj        dz  c_        t                               d¦  «         t          j        t          | j        t          ¦  «        fcd d d ¦  «         S | j                             ¦   «         | j        k    r\| xj        dz  c_        t                               d¦  «         t          j        t          | j        | j        ¦  «        fcd d d ¦  «         S | j                             ¦   «         dk    rV| xj        dz  c_        t                               d¦  «         t          j        t/          | j        ¦  «        fcd d d ¦  «         S | j                             ¦   «         dk    r\| xj        dz  c_        t                               d¦  «         t          j        t          | j        t          ¦  «        fcd d d ¦  «         S | j                             ¦   «         dk    r\| xj        dz  c_        t                               d¦  «         t          j        t          | j        | j        ¦  «        fcd d d ¦  «         S | j        dk    r¬| j                             ¦   «         dk    r�| j        �ˆt          j        ¦   «         | j        z
  dk    ri| xj        dz  c_        t                               d¦  «         t          j	        t          | j        |                      ¦   «         ¦  «        fcd d d ¦  «         S | j        dk    rþ| j                             ¦   «         dk    rá| j                             ¦   «         dk    rÄ| j                             ¦   «         dk    r§| j                             ¦   «         dk    rŠ| j        dk    r| j        dk    rt| j        dk    ri| xj        dz  c_        t                               d¦  «         t          j	        t          | j        |                      ¦   «         ¦  «        fcd d d ¦  «         S t1          d„ | j        D ¦   «         ¦  «        r(t                               d¦  «         	 d d d ¦  «         d S | xj        dz  c_        t                               dt8          › d�¦  «         t          j        g fcd d d ¦  «         S # 1 swxY w Y   d S )Nr   i,  r	   z;Job: commit (more than 5 minutes since last commit attempt)é–   zJob: commit (>100 files ready)zJob: get upload mode (>z files ready)z5Job: preupload LFS (no other worker preuploading LFS)z.Job: sha256 (no other worker computing sha256)z:Job: get upload mode (no other worker getting upload mode)zJob: preupload LFSzJob: sha256zJob: get upload moderg   z-Job: commit (1 min since last commit attempt)zJob: commitc              3   ó8   K  — | ]\  }}|j         p|j        V — Œd S rx   r  r  s      r0   r=   z&_determine_next_job.<locals>.<genexpr>™  s1   è è € ÐaÐaÁ[ÀQÈ�Ô&Ð@¨(Ô*@ÐaÐaÐaÐaÐaÐar/   z.All files have been processed! Exiting worker.zNo task available, waiting... (zs))rÅ   rÉ   rÄ   ÚqsizerË   r�   rB   r”   r±   rº   Ú_get_nrá   rÂ   ÚMAX_NB_FILES_FETCH_UPLOAD_MODErÇ   r¸   rÃ   rª   rÈ   r¹   rÁ   rÆ   r·   Ú_get_oner  rO   rC   rÊ   r  r»   )r‚   s    r0   r  r  :  s,	  € Ø	Œð f(ð f(ð Ô$¨Ò)Ð)ØÔ#×)Ò)Ñ+Ô+¨aÒ/Ð/ØÔ*Ð6Ý”	‘”˜fÔ8Ñ8¸6ÒAÐAàÐ$Ô$¨Ñ)Ð$Ô$Ý�LŠLÐVÑWÔWÐWÝÔ$¥f¨VÔ-@À&×BUÒBUÑBWÔBWÑ&XÔ&XÐYðf(ð f(ð f(ð f(ñ f(ô f(ð f(ð f(ð Ô%¨Ò*Ð*¨vÔ/B×/HÒ/HÑ/JÔ/JÈcÒ/QÐ/QØÐ$Ô$¨Ñ)Ð$Ô$Ý�LŠLÐ9Ñ:Ô:Ð:ÝÔ$¥f¨VÔ-@À&×BUÒBUÑBWÔBWÑ&XÔ&XÐYð!f(ð f(ð f(ð f(ñ f(ô f(ð f(ð f(ð& Ô)×/Ò/Ñ1Ô1Õ5SÒSÐSØÐ-Ô-°Ñ2Ð-Ô-Ý�LŠLÐ`Õ3QÐ`Ð`Ð`ÑaÔaÐaÝÔ-­v°fÔ6RÕTrÑ/sÔ/sÐtð-f(ð f(ð f(ð f(ñ f(ô f(ð f(ð f(ð2 Ô'×-Ò-Ñ/Ô/°6Ô3KÒKÐKÐPVÔPoÐstÒPtÐPtØÐ+Ô+¨qÑ0Ð+Ô+Ý�LŠLÐPÑQÔQÐQÝÔ+­V°FÔ4NÐPVÔPhÑ-iÔ-iÐjð9f(ð f(ð f(ð f(ñ f(ô f(ð f(ð f(ð> Ô ×&Ò&Ñ(Ô(¨1Ò,Ð,°Ô1IÈQÒ1NÐ1NØÐ$Ô$¨Ñ)Ð$Ô$Ý�LŠLÐIÑJÔJÐJÝÔ$¥h¨vÔ/BÑ&CÔ&CÐDðEf(ð f(ð f(ð f(ñ f(ô f(ð f(ð f(ðJ Ô)×/Ò/Ñ1Ô1°AÒ5Ð5¸&Ô:[Ð_`Ò:`Ð:`ØÐ-Ô-°Ñ2Ð-Ô-Ý�LŠLÐUÑVÔVÐVÝÔ-­v°fÔ6RÕTrÑ/sÔ/sÐtðQf(ð f(ð f(ð f(ñ f(ô f(ð f(ð f(ðV Ô'×-Ò-Ñ/Ô/°6Ô3KÒKÐKØÐ+Ô+¨qÑ0Ð+Ô+Ý�LŠLÐ-Ñ.Ô.Ð.ÝÔ+­V°FÔ4NÐPVÔPhÑ-iÔ-iÐjð]f(ð f(ð f(ð f(ñ f(ô f(ð f(ð f(ðb Ô ×&Ò&Ñ(Ô(¨1Ò,Ð,ØÐ$Ô$¨Ñ)Ð$Ô$Ý�LŠL˜Ñ'Ô'Ð'ÝÔ$¥h¨vÔ/BÑ&CÔ&CÐDðif(ð f(ð f(ð f(ñ f(ô f(ð f(ð f(ðn Ô)×/Ò/Ñ1Ô1°AÒ5Ð5ØÐ-Ô-°Ñ2Ð-Ô-Ý�LŠLÐ/Ñ0Ô0Ð0ÝÔ-­v°fÔ6RÕTrÑ/sÔ/sÐtðuf(ð f(ð f(ð f(ñ f(ô f(ð f(ð f(ðz Ô'×-Ò-Ñ/Ô/°!Ò3Ð3ØÐ+Ô+¨qÑ0Ð+Ô+Ý�LŠLÐ-Ñ.Ô.Ð.ÝÔ+­V°FÔ4NÐPVÔPhÑ-iÔ-iÐjðAf(ð f(ð f(ð f(ñ f(ô f(ð f(ð f(ðH Ô$¨Ò)Ð)ØÔ#×)Ò)Ñ+Ô+¨aÒ/Ð/ØÔ*Ð6Ý”	‘”˜fÔ8Ñ8¸6ÒAÐAàÐ$Ô$¨Ñ)Ð$Ô$Ý�LŠLÐHÑIÔIÐIÝÔ$¥f¨VÔ-@À&×BUÒBUÑBWÔBWÑ&XÔ&XÐYðWf(ð f(ð f(ð f(ñ f(ô f(ð f(ð f(ð` Ô$¨Ò)Ð)ØÔ#×)Ò)Ñ+Ô+¨aÒ/Ð/ØÔ#×)Ò)Ñ+Ô+¨qÒ0Ð0ØÔ,×2Ò2Ñ4Ô4¸Ò9Ð9ØÔ*×0Ò0Ñ2Ô2°aÒ7Ð7ØÔ(¨AÒ-Ð-ØÔ1°QÒ6Ð6ØÔ/°1Ò4Ð4àÐ$Ô$¨Ñ)Ð$Ô$Ý�LŠL˜Ñ'Ô'Ð'ÝÔ$¥f¨VÔ-@À&×BUÒBUÑBWÔBWÑ&XÔ&XÐYðwf(ð f(ð f(ð f(ñ f(ô f(ð f(ð f(õ| ÐaÐaÐTZÔT`ÐaÑaÔaÑaÔað 	(Ý�KŠKÐHÑIÔIÐIØðAf(ð f(ð f(ñ f(ô f(ð f(ð f(ð f(ðH Ð%Ô%¨Ñ*Ð%Ô%Ý�LŠLÐWÕ;SÐWÐWÐWÑXÔXÐXÝ”N BÐ'ðMf(ð f(ð f(ð f(ñ f(ô f(ð f(ð f(ð f(ð f(ð f(ð f(øøøð f(ð f(ð f(ð f(ð f(ð f(sl   ˆB+^Ã B^ÅA:^ÇA<^É!A1^ËA7^Í#A1^Ï!A&^ÑA,^ÓA,^ÕB*^×=C<^Ü9^ÝA ^Þ^Þ ^r×   c                 óø   — | \  }}|j         €X|j                             d¦  «        5 }t          |¦  «                             ¦   «         |_         ddd¦  «         n# 1 swxY w Y   |                     |¦  «         dS )z1Compute sha256 of a file and save it in metadata.NÚrb)rÑ   rQ   Úopenr   ÚhexÚsave)r×   rX   rØ   Úfs       r0   r  r  ©  s¬   € à�O€Eˆ8Ø„ÐØŒ_×!Ò! $Ñ'Ô'ð 	3¨1Ý)¨!™nœn×0Ò0Ñ2Ô2ˆHŒOð	3ð 	3ð 	3ñ 	3ô 	3ð 	3ð 	3ð 	3ð 	3ð 	3ð 	3øøøð 	3ð 	3ð 	3ð 	3à‡M‚M�%ÑÔÐÐÐs   §'AÁAÁ!ArO   c           
      ó<  — d„ | D ¦   «         }t          ||||                     ¦   «         t          |d¬¦  «        |j        ¬¦  «         t	          | |¦  «        D ]C\  }}|\  }}	|j        |	_        |j        |	_        |j	        |	_
        |	                     |¦  «         ŒDdS )zmGet upload mode for each file and update metadata.

    Also receive info if the file should be ignored.
    c                 ó,   — g | ]}t          |¦  «        ‘ŒS r.   ©Ú_build_hacky_operation©r:   r×   s     r0   r~   z$_get_upload_mode.<locals>.<listcomp>·  ó!   € Ð@Ð@Ð@°$Õ'¨Ñ-Ô-Ð@Ð@Ð@r/   r?   )Úsafe)Ú	additionsrr   rp   Úheadersrh   ÚendpointN)r   Ú_build_hf_headersr   r/  ÚzipÚ_upload_moderÓ   Ú_should_ignorerì   Ú_remote_oidÚ
remote_oidr$  )
rO   ro   rp   rr   rh   r-  r×   ÚadditionrX   rØ   s
             r0   r  r  ²  sÄ   € ð
 AÐ@¸%Ð@Ñ@Ô@€IÝØØØØ×%Ò%Ñ'Ô'Ý�x bÐ)Ñ)Ô)Ø”ðñ ô ð õ ˜e YÑ/Ô/ð ð ‰ˆˆhØ‰ˆˆxØ'Ô4ˆÔØ!)Ô!8ˆÔØ&Ô2ˆÔØ�Š�eÑÔÐÐðð r/   c                 ó˜   — d„ | D ¦   «         }|                      ||||¬¦  «         | D ]!\  }}d|_        |                     |¦  «         Œ"dS )z(Preupload LFS files and update metadata.c                 ó,   — g | ]}t          |¦  «        ‘ŒS r.   r(  r*  s     r0   r~   z"_preupload_lfs.<locals>.<listcomp>Ê  r+  r/   )rp   rr   rh   r-  TN)Úpreupload_lfs_filesrÔ   r$  ©rO   ro   rp   rr   rh   r-  rX   rØ   s           r0   r  r  È  sz   € à@Ð@¸%Ð@Ñ@Ô@€IØ×ÒØØØØð	 ñ ô ð ð !ð ð ‰ˆˆxØ#ˆÔØ�Š�eÑÔÐÐðð r/   c                 óš   — d„ | D ¦   «         }|                      ||||d¬¦  «         | D ]!\  }}d|_        |                     |¦  «         Œ"dS )zCommit files to the repo.c                 ó,   — g | ]}t          |¦  «        ‘ŒS r.   r(  r*  s     r0   r~   z_commit.<locals>.<listcomp>Ù  r+  r/   z(Add files using upload-large-folder tool)rp   rr   rh   Ú
operationsÚcommit_messageTN)Úcreate_commitrÕ   r$  r:  s           r0   r  r  ×  s}   € à@Ð@¸%Ð@Ñ@Ô@€IØ×ÒØØØØØAð ñ ô ð ð !ð ð ‰ˆˆxØ $ˆÔØ�Š�eÑÔÐÐðð r/   c                   ó   — e Zd Zdd„ZdS )ÚHackyCommitOperationAddr&   Nc                 óp   — t          | j        t          ¦  «        rt          | j        ¦  «        | _        d S d S rx   )r�   Úpath_or_fileobjr   rJ   rÛ   s    r0   Ú__post_init__z%HackyCommitOperationAdd.__post_init__ì  s:   € Ý�dÔ*­DÑ1Ô1ð 	=Ý#& tÔ';Ñ#<Ô#<ˆDÔ Ð Ð ð	=ð 	=r/   r  )r²   r³   r´   rD  r.   r/   r0   rA  rA  ë  s(   € € € € € ð=ð =ð =ð =ð =ð =r/   rA  c                 ó  — | \  }}t          |j        |j        ¬¦  «        }|j                             d¦  «        5 }|                     d¦  «        d d…         }d d d ¦  «         n# 1 swxY w Y   |j        €t          d¦  «        ‚t          t           	                    |j        ¦  «        |j
        |¬¦  «        |_        |j        |_        |j        |_        |j        |_        |j        |_        |j        r|j        dk    rd|_        |S )N)rH   rC  r!  i   z&sha256 must have been computed by now!)rÑ   r<   Úsampler¾   r/   )rA  rH   rQ   r"  ÚpeekrÑ   r‰   r   ÚbytesÚfromhexr<   Úupload_inforÓ   r2  rì   r3  r5  r4  rÔ   Ú_is_uploadedrC  )r×   rX   rØ   Ú	operationÚfilerF  s         r0   r)  r)  ñ  s7  € Ø�O€Eˆ8Ý'°UÔ5GÐY^ÔYhÐiÑiÔi€IØ	Œ×	Ò	˜dÑ	#Ô	#ð & tØ—’˜3‘”   Ô%ˆð&ð &ð &ñ &ô &ð &ð &ð &ð &ð &ð &øøøð &ð &ð &ð &à„ÐÝÐAÑBÔBÐBÝ&­e¯mªm¸H¼OÑ.LÔ.LÐS[ÔS`ÐioÐpÑpÔp€IÔØ%Ô1€IÔØ'Ô5€IÔØ$Ô/€IÔØ%Ô1€IÔØÔð ( Ô 4¸Ò =Ð =Ø$'ˆ	Ô!ØÐs   »A%Á%A)Á,A)r¿   zqueue.Queue[JOB_ITEM_T]c                 ó"   —  | j         ¦   «         gS rx   ©Úget)r¿   s    r0   r  r    s   € ØˆEŒI‰KŒKˆ=Ðr/   Únc                 óp   ‡ — ˆ fd„t          t           ‰ j        ¦   «         |¦  «        ¦  «        D ¦   «         S )Nc                 ó.   •— g | ]} ‰j         ¦   «         ‘ŒS r.   rO  )r:   rˆ   r¿   s     €r0   r~   z_get_n.<locals>.<listcomp>  s!   ø€ Ð>Ð>Ð>˜AˆIˆEŒI‰KŒKÐ>Ð>Ð>r/   )r™   rç   r  )r¿   rQ  s   ` r0   r  r    s7   ø€ Ø>Ð>Ð>Ð>¥¥s¨;¨5¬;©=¬=¸!Ñ'<Ô'<Ñ!=Ô!=Ð>Ñ>Ô>Ð>r/   Úreportc                 óJ  ‡— | dz  } t          j        ¦   «         j        Št          ˆfd„|                      ¦   «         D ¦   «         ¦  «        }t          |¦  «        D ]@}t          j                             d¦  «         t          j                             d¦  «         ŒAt          j                             | ¦  «         t          j                             d‰t          |                      ¦   «         d         ¦  «        z
  z  ¦  «         t          j         
                    ¦   «          dS )z÷Print a report, overwriting the previous lines.

    Since tqdm in using `sys.stderr` to (re-)write progress bars, we need to use `sys.stdout`
    to print the report.

    Note: works well only if no other process is writing to `sys.stdout`!
    rv   c              3   óB   •K  — | ]}t          |¦  «        ‰z  d z   V — ŒdS )r	   N)rD   )r:   ÚlineÚterminal_widths     €r0   r=   z#_print_overwrite.<locals>.<genexpr>  s3   øè è € ÐSÐS°t•3�t‘9”9 Ñ.°Ñ2ÐSÐSÐSÐSÐSÐSr/   z[Kz[Frê   r3   N)ÚshutilÚget_terminal_sizeÚcolumnsÚsumÚ
splitlinesr™   ÚsysÚstdoutÚwriterD   Úflush)rT  Únb_linesrˆ   rX  s      @r0   rŸ   rŸ     s   ø€ ð ˆd�N€FåÔ-Ñ/Ô/Ô7€Nõ ÐSÐSÐSÐS¸v×?PÒ?PÑ?RÔ?RÐSÑSÔSÑSÔS€Hõ �8‰_Œ_ð #ð #ˆÝŒ
×Ò˜Ñ$Ô$Ð$ÝŒ
×Ò˜Ñ"Ô"Ð"Ð"õ „J×Ò�VÑÔÐÝ„J×Ò�S˜N­S°×1BÒ1BÑ1DÔ1DÀRÔ1HÑ-IÔ-IÑIÑJÑKÔKÐKÝ„J×ÒÑÔÐÐÐr/   )Mrµ   ÚloggingrŽ   r¿   rY  r^  r…   r�   r  r   Úpathlibr   r   Útypingr   r   Úurllib.parser   Ú_commit_apir
   r   r   Ú_local_folderr   r   r   r   Ú	constantsr   r   Úutilsr   r   r   r   Úutils._runtimer   Ú	utils.shar   Úhf_apir   Ú	getLoggerr²   rB   r  r¢   r  r#   r  r  Ú__annotations__r•   r–   rE   rP   rT   rV   rf   rJ   r	  r¯   ÚEnumr±   Útupler  r˜   r‡   r  r  r  r  r  rA  r)  r  r  rŸ   r.   r/   r0   ú<module>rr     s´  ðð €€€€Ø €€€Ø 	€	€	€	Ø €€€Ø €€€Ø 
€
€
€
Ø Ð Ð Ð Ø €€€Ø Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø %Ð %Ð %Ð %Ð %Ð %Ð %Ð %Ø Ð Ð Ð Ð Ð à LÐ LÐ LÐ LÐ LÐ LÐ LÐ LÐ LÐ LØ vÐ vÐ vÐ vÐ vÐ vÐ vÐ vÐ vÐ vÐ vÐ vØ 3Ð 3Ð 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SØ ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø "Ð "Ð "Ð "Ð "Ð "ð ð ØÐÐÐÐÐà	ˆÔ	˜8Ñ	$Ô	$€àÐ ØÐ Ø!$Ð ØOÐOÐOÐ �4˜”9Ð OÐ OÑ OàÐ ØÐ ð Ð ØÐ ØÐ ØÐ ð]/¨Ð-AÔ(Bð ]/Àtð ]/ð ]/ð ]/ð ]/ðL  ØØ-1Ø.2Ø"ØØ ðL'ð L'ð L'Ø	ðL'àðL'ð �t‘ðL'ð
 ðL'ð �D‰jðL'ð �D‰[ðL'ð ˜”I ‘O dÑ*ðL'ð ˜#”Y ‘_ tÑ+ðL'ð �t‘ðL'ð ðL'ð ðL'ð L'ð L'ð L'ðhð ð ð ð �”	ñ ô ð ð Ð'Ð)@Ð@ÔA€
ðFfð Ffð Ffð Ffð Ffñ Ffô Ffð FfðRv3Øðv3à	ðv3ð ðv3ð ð	v3ð
 ðv3ð v3ð v3ð v3ðrg(Ð 1ð g(°e¸IÀtÈJÔGWÐ<WÔ6XÐ[_Ñ6_ð g(ð g(ð g(ð g(ð^˜*ð ¨ð ð ð ð ð˜D Ô,ð °7ð ÀSð ÐUXð Ðdgð Ðlpð ð ð ð ð,˜$˜zÔ*ð °ð À3ð ÐSVð Ðbeð Ðjnð ð ð ð ð�4˜
Ô#ð ¨'ð ¸Cð ÈCð Ð[^ð Ðcgð ð ð ð ð(=ð =ð =ð =ð =Ð0ñ =ô =ð =ð ð Ð0Gð ð ð ð ð,Ð-ð °$°zÔ2Bð ð ð ð ð?Ð+ð ?°ð ?¸¸ZÔ8Hð ?ð ?ð ?ð ?ð˜Sð  Tð ð ð ð ð ð r/   