"j&( dZddlZddlZddlZddlmZddlmZddlZddlm Z ddl m Z m Z m Z mZmZmZmZddlmZmZmZmZmZdd lmZmZmZmZmZmZdd lm Z m!Z!e j"e#Z$ed Z%d e&de'fdZ(de&de)e&e&ffdZ*dde+e,zde'de&fdZ-ddedzde'de&fdZ. dde/e e zde'de'de/e&fdZ0 dde1de/e&de&de+de+de'ddfd Z2e%3d!gd"#ddde j4fd$ee&ej5d% fd&ee'ej6d'd( fd)ee'ej6d*d+ fd,ed-eddf d.Z7de&de'fd/Z8e%3d0gd1#dddddde j4fdee&dzej5d2 fdee'ej6d3d4d5 fd6ee'ej6d7d8 fd9ee'ej6d:d;d< fd=ed,ed-eddfd>Z9d?e&dzd=e&dzde'd6e'd9e'd,e&dzddfd@Z:de&de'd6e'd9e'd,e&dzddf dAZ;e%3dBdCdDg#de j4fd$ee&ej5dE fd,ed-eddfdFZe%3d]gd^#de j4fd_ee&ej5d` fdaee&ej5db fd,ed-eddf dcZ?e%3ddgde# ddfee&dzej5dg fdhee&dzej5di fdGee'ej6dj fdkee'ej6dldm fdnee'ej6dodp fdqee&dzej6dr fdsee&dzej6dt fdUee'ej6dVdu fdXee/e&dzej6dv fdZee/e&dzej6dw fdxee&dzej6dy fdzee'ej6d{d| fd}ee'ej6d~d fdee'ej6ddd fdee'ej6ddd fd,eddf"dZ@e%3dgd#dde j4fdee&ej5d fdee&dzej5d fd,ed-eddf dZAdS)z7Contains commands to interact with buckets via the CLI.N)datetime) Annotated)logging) BUCKET_PREFIX BucketFile BucketFolder FilterMatcher_is_bucket_path_parse_bucket_path_split_bucket_id_and_prefix)SoftTemporaryDirectory StatusLineare_progress_bars_disableddisable_progress_barsenable_progress_bars)FormatWithAutoOpt SearchOptTokenOptapi_object_to_dict get_hf_api typer_factory)OutputFormatWithAutooutz"Commands to interact with buckets.)helppathreturnc,|dS)Nzhf://) startswithrs e/lsinfo/ai/hellotax_ai/base_platform/venv/lib/python3.11/site-packages/huggingface_hub/cli/buckets.py _is_hf_handler";s ??7 # ##argumentc|trt|S t|S#t$rt d|dtdwxYw)zParse a bucket argument accepting both 'namespace/name(/prefix)' and 'hf://buckets/namespace/name(/prefix)'. Returns: tuple: (bucket_id, prefix) where bucket_id is "namespace/bucket_name" and prefix may be empty string. zInvalid bucket argument: -. Must be in format namespace/bucket_name or znamespace/bucket_name)rrr r ValueError)r$s r!_parse_bucket_argumentr(?s =)),!(+++ *8444     8 8 8  8 8 8    s :&A Fsizehuman_readablec|st|SdD]&}|dkr|dkr |d|cS|dd|cS|dz}'|ddS)zFormat a size in bytes.)BKBMBGBTBir, z.1fz PB)str)r)r*units r! _format_sizer4Ps 4yy- $;;s{{'''''''''''' ' ' '      r#mtimecb|dS|r|dS|dS)z0Format mtime datetime to a readable date string.Nz %b %d %H:%Mz%Y-%m-%d %H:%M:%S)strftime)r5r*s r! _format_mtimer9^s9 }r-~~m,,, >>- . ..r#itemsquietcNi}|D]t}|jd}|}|ddD]}||vrdii||<||d}|d}t|tr ||vrdii||<md|i||<ud} d} d} |s|D]{}t|trdt |j|} t| t| } t|j |} t| t| } || dkr| dz| z} g}t||d| | | |S) aBuild a tree representation of files and directories. Produces ASCII tree with size and date columns before the tree connector. When quiet=True, only the tree structure is shown (no size/date). Args: items: List of BucketFile/BucketFolder items human_readable: Whether to show human-readable sizes and short dates quiet: If True, show only the tree structure without sizes/dates Returns: List of formatted tree lines /N __children____item__rr7 prefix_widthmax_size_widthr*) rsplit isinstancerrr4r)maxlenr9r5 _render_tree)r:r*r;treeitempartscurrentpart final_partrCrDmax_date_widthsize_strdate_strliness r! _build_treerTgs&D 5 5 $$#2#J 4 4D7""!/ 4 dmN3GG2Y dL ) ) 5(('5r&: ##-t"4GJ  LNN ? D DD$ ++ D' >BB!$^S]]!C!C(^DD!$^S]]!C!C A  )A->LE   !%%  Lr#noderSindentrCrDc tt|}t|D]\}\}} |t|dz k} | rdnd} d| v} | di} |dkr| rd|z}nR| d}|6t |j|}t|j|}|d |d |}nd|z}| |d || || rd nd n!| || || rd nd | r|| rd ndz}t| |||||dS)z:Recursively render a tree structure with size+date prefix.ru └── u ├── r?rr1r@N> r=r7z u│ rB) sortedr: enumeraterHgetr4r)r9r5appendrI)rUrSrVrCrDr*r:inamevalueis_last connectoris_dirchildrenprefixrKrQrR child_indents r!rIrIs 4::<< E%e,,=D%s5zzA~%$+=LL 5(99^R00 !   0|+yy,,#+DI~FFH,TZHHH (H>HHHHhHHFF </F LLFVVfViVVf?TssRTVV W W W W LLFLILtLF5JSSLL M M M  !w%DVVHEL )--     1r#create)zhf buckets create my-bucketz hf buckets create user/my-bucketz-hf buckets create hf://buckets/user/my-bucketz*hf buckets create user/my-bucket --privatez+hf buckets create user/my-bucket --exist-ok)r_examples bucket_idzTBucket ID: bucket_name, namespace/bucket_name, or hf://buckets/namespace/bucket_nameprivatez --privatezCreate a private bucket.exist_okz --exist-okz3Do not raise an error if the bucket already exists.tokenformatct|}|trk t|\}}n3#t$r&}t jt|d}~wwxYw|r t jd|dtd|}|||r|nd|} tj d| j | j dS) zCreate a new bucket.rlNz-Cannot specify a prefix for bucket creation: . Use namespace/bucket_name or namespace/bucket_name.)rjrkzBucket created)handleurl) rrrr(r'typer BadParameterr2 create_bucketrresultrrrs) rirjrkrlrmapi parsed_idree bucket_urls r!rgrgsF 5 ! ! !CM**  - 6y A A Ivv - - -$SVV,, , -  $W WW1>WWW  """,#J J (9z~NNNNNNs? A/ !A**A/c||tr|ttd}n|}d|vS)zFCheck if argument is a bucket ID (namespace/name) vs just a namespace.Nr=)rrrH)r$rs r! _is_bucket_idr} sA=))M**,,- $;r#z list | ls) zhf buckets listzhf buckets list huggingfacez$hf buckets list --search "my-prefix"zhf buckets list user/my-bucketz!hf buckets list user/my-bucket -Rz!hf buckets list user/my-bucket -hz%hf buckets list user/my-bucket --treez(hf buckets list user/my-bucket --tree -hz+hf buckets list hf://buckets/user/my-bucketz%hf buckets list user/my-bucket/sub -RzyNamespace (user or org) to list buckets, or bucket ID (namespace/bucket_name(/prefix) or hf://buckets/...) to list files.z--human-readablez-hz$Show sizes in human readable format.as_treez--treez3List files in tree format (only for listing files). recursivez --recursivez-Rz0List files recursively (only for listing files).searchc|duot|}|r,|tjdt|||||dSt ||||||dS)zList buckets or files in a bucket. When called with no argument or a namespace, lists buckets. When called with a bucket ID (namespace/bucket_name), lists files in the bucket. Nz'Cannot use --search when listing files.)r$r*r~rrl) namespacerr*r~rrl)r}rtru _list_files _list_buckets)r$r*r~rrrlrm is_file_modes r!list_cmdrst4'CM(,C,CL  $%NOO O)        )       r#rc|rtjd|rtjd|K|tr1|t td}|d}t |}fd|||D}tj |ddd  dS) zList buckets in a namespace.z'Cannot use --tree when listing buckets.z,Cannot use --recursive when listing buckets.Nr=rocg|]:}|j|jrt|jn|j|j|jd;S))idrjr) total_files created_at)rrjr4r)rr).0bucketr*s r! z!_list_buckets..{se     )~AO`Ln===U[U`!- +      r#)rrright)r)r) alignments) rtrurrrHrstripr list_bucketsrtable)rrr*r~rrlrxr:s ` r!rresL !JKKKQ !OPPP!5!5m!D!Dc-00223 $$S)) 5 ! ! !C    &&6&JJ   EIe I IJJJJJJr#c |r.tjtjkrt jdt |} t|\}}n3#t$r&}t jt|d}~wwxYwt| ||pd|} | stj ddStd| D} |r@tjtjk} t| || D]} t!| n@tjtjkr.t!tjd| Dd ntjtjkrG| D]C} t%| t&rt!| jd /t!| jDn| D]} t%| t&r5t+| j|}t!d d d|dd| jd Lt/| j|}t+| j|}t!|d d|dd| j|s| rtjddSdSdS)zList files in a bucket.z%Cannot use --tree with --format json.roNrerz(empty)c3@K|]}t|tVdSN)rFrrrKs r! z_list_files..s,KKT*T<88KKKKKKr#)r*r;c,g|]}t|S)rrs r!rz_list_files..s!EEEt,T22EEEr#rA)rVr=r7z>12rYz>19z!Use -R to list files recursively.)rmoderjsonrtrurr(r'r2listlist_bucket_treetextanyr;rTprintdumpsrFrrr9 uploaded_atr4r)r5hint)r$r*r~rrlrxrirerzr:has_directoriesr;linerK mtime_strrQs r!rrsJ383888 !HIII 5 ! ! !C)28<< 66 ))) Q((()   >T     E  KKUKKKKKOG066nERRR  D $KKKK  ). . . djEEuEEEaPPPQQQQ )/ / / ! !D$ -- !ooo&&&&di     ! G GD$ -- G)$*:NKK @@@9@@@DI@@@AAAA' >BB)$*nEE EEEEEE$)EEFFFF 66 4555556666sA B!BBinfozhf buckets info user/my-bucketz+hf buckets info hf://buckets/user/my-bucketzFBucket ID: namespace/bucket_name or hf://buckets/namespace/bucket_namec t|} t|\}}n3#t$r&}tjt |d}~wwxYw||}tj|ddS)zGet info about a bucket.roNr)id_key) rr(r'rtrur2 bucket_infordict)rirlrmrxry_rzrs r!rrs$ 5 ! ! !C)-i88 11 ))) Q((()__Y ' 'FHVD!!!!!!s% A!AAdelete)z hf buckets delete user/my-bucketz-hf buckets delete hf://buckets/user/my-bucketz&hf buckets delete user/my-bucket --yesz-hf buckets delete user/my-bucket --missing-okyesz--yesz-yzSkip confirmation prompt. missing_okz --missing-okz3Do not raise an error if the bucket does not exist.c|trl t|\}}n3#t$r&}t jt |d}~wwxYw|r t jd|dtd|}n$d|vr t jd|dtdtjd|d | t| }| || tj d |dS)zDelete a bucket. This deletes the entire bucket and all its contents. Use `hf buckets rm` to remove individual files. Nz-Cannot specify a prefix for bucket deletion: rprqr=zInvalid bucket ID: r&z(Are you sure you want to delete bucket ''?rro)rzBucket deleted)ri) rrr(r'rtrur2rconfirmr delete_bucketrw) rirrrlrmryrerzrxs r!rrsjLM**  - 6y A A Ivv - - -$SVV,, , -  $W WW1>WWW  I    a) a a;H a a a   KH9HHHcRRRR 5 ! ! !CiJ777J9555555s/ A!AAz remove | rm)z)hf buckets remove user/my-bucket/file.txtz2hf buckets rm hf://buckets/user/my-bucket/file.txtz.hf buckets rm user/my-bucket/logs/ --recursivez:hf buckets rm user/my-bucket --recursive --include "*.tmp"z8hf buckets rm user/my-bucket/data/ --recursive --dry-runzBucket path: namespace/bucket_name/path or hf://buckets/namespace/bucket_name/path. With --recursive, namespace/bucket_name is also accepted to target all files.z0Remove files recursively under the given prefix.dry_runz --dry-runz8Preview what would be deleted without actually deleting.includezQInclude only files matching pattern (can specify multiple). Requires --recursive.excludezLExclude files matching pattern (can specify multiple). Requires --recursive.c> t|\}n3#t$r&} tjt | d} ~ wwxYw|dkr|stjddd|s|r|stjdt |} |rSt tjtj k} | d g} | | d pdd D]R} t| tr;| | | d t#| dS| d t#| d|s|r t'||fd| D}n| }d|D}t)d|D}t+|d }|stjddSt#|d|}|sP|sNtjdd|Dtjd|ddd|rMtjdfd|Dtjd|ddS| | tjd!|dd"t#||#dS| d }|stjd$|r7tjd%t6d |tjd&dStjd'|d(d|| |g tjd)|*dS)+zbRemove files from a bucket. To delete an entire bucket, use `hf buckets delete` instead. Nr7z?No file path specified. To remove files, provide a path (e.g. 'zd/FILE') or use --recursive to remove all files. To delete the entire bucket, use `hf buckets delete z`.z,--include and --exclude require --recursive.ro)enabledzListing files from remoter=TrzListing files from remote (z files))include_patternsexclude_patternscHg|]}|j|Sr)matchesr)rfmatchers r!rzremove..s,MMM1W__QV5L5LMQMMMr#cg|] }|j Srr rrs r!rzremove..s444af444r#c3$K|] }|jV dSr)r)rs r!rzremove..s$77A777777r#)r*zNo files to remove.z file(s) totaling  c3 K|] }d|V dS)rYNr)rrs r!rzremove..s(BBt{D{{BBBBBBr#zRemove z from 'rFrc36K|]}dtd|VdS)delete: r=N)r)rrris r!rzremove..s:ccPTK-KKKTKKccccccr#z (dry run) z would be removed.)rzRemoved ')ri files_deletedr)zFile path cannot be empty.rz"(dry run) 1 file would be removed.zRemove 'z' from 'z File removed)rri)r(r'rtrur2rrrrrhumanupdaterrrFrr]rHdoner sumr4rjoinrbatch_bucket_filesrwr)r$rrrrrrlrmrerzrxstatus all_filesrK matched_files file_paths total_sizerQ count_label file_pathrirs @@r!remover!sz|)28<< 66 ))) Q((()||I|  R  R RDM R R R   Q7QIQ !OPPP 5 ! ! !C=HCH0D0J$JKKK 1222&( (( ==%%-)   U UD $ ++ U  &&& SC NNSSSTTT I#i..IIIJJJ  &g &#WwWWWGMMMM MMMMM%M44m444 7777777  4@@@  H* + + + FZFFHFF  P7 P HTYYBBzBBBBB C C C KC+CCiCCC O O O O   HTYYccccXbccccc d d d HA+AAA B B B F y<<< 7{ 7 79 7 7 7j//       MM#&&  C$%ABB B   HF FyFF9FF G G G H9 : : : F ?y??)???SIIII y)=== > YGGGGGGs A!AAmove)z/hf buckets move user/old-bucket user/new-bucketz/hf buckets move user/my-bucket my-org/my-bucketzIhf buckets move hf://buckets/user/old-bucket hf://buckets/user/new-bucketfrom_idzMSource bucket ID: namespace/bucket_name or hf://buckets/namespace/bucket_nameto_idzRDestination bucket ID: namespace/bucket_name or hf://buckets/namespace/bucket_namecRt|\}}|r tjd|dtdt|\}}|r tjd|dtdt |}|||t jd||dS)z2Move (rename) a bucket to a new name or namespace.z)Cannot specify a prefix for bucket move: rprqro)rrz Bucket movedN)r(rtrurr move_bucketrrw) rrrlrmparsed_from_id from_prefix parsed_to_id to_prefixrxs r!rrs4#9"A"ANK   S S S-: S S S   5U;;L)   S S S-: S S S   5 ! ! !COON,O???J~~\JJJJJJr#sync)z2hf buckets sync ./data hf://buckets/user/my-bucketz2hf buckets sync hf://buckets/user/my-bucket ./dataz;hf buckets sync ./data hf://buckets/user/my-bucket --deletez^hf buckets sync hf://buckets/user/my-bucket ./data --include "*.safetensors" --exclude "*.tmp"zIhf buckets sync ./data hf://buckets/user/my-bucket --plan sync-plan.jsonlz'hf buckets sync --apply sync-plan.jsonlzhf buckets cp my-config.json hf://buckets/user/my-bucket/logs/zKhf buckets cp my-config.json hf://buckets/user/my-bucket/remote-config.jsonz7hf buckets cp - hf://buckets/user/my-bucket/config.jsonzVhf buckets cp hf://buckets/user/my-bucket/logs/ hf://buckets/user/archive-bucket/logs/zehf buckets cp hf://datasets/user/my-dataset/processed/ hf://buckets/user/my-bucket/dataset/processed/srczMSource: local file, any hf:// handle (model, dataset, bucket), or - for stdindstz@Destination: local path, bucket hf://... handle, or - for stdoutc t|}t|}|duot|}t|}|duot|}|dk} |dk} |rf|rd |||n3#t$r&} t jt| d} ~ wwxYwtj d||dS|s.|s,| s*|t jdt jd| r|st jd| rC|rAt|\} } | d ks| d rt jd | r|st jd |s5| s3tj |rt jd |rft|\}} | d ks| d rt jd| d dd}| r"t!}|st# t%5}tj ||}||| |fgt+|d5}|dx}r;t.jj||dx};dddn #1swxYwYdddn #1swxYwY|st7dSdS#|st7wwxYw||}nvtj |s4| tjs| d r!tj ||}n|}tj |}|rtj|d||| |fgtj d||dS| rht|\}}t.jj}| |||fgtj dd|dStj !|st jd|t|\}} | d kr tj "|}n:| d r#| tj "|z}n| }| |||fgtj d|tF|d |dS)zCopy files to or from buckets.roN-Copied)rrz2Missing destination. Provide a bucket path as DST.z;One of SRC or DST must be a bucket path (hf://buckets/...).z+Stdin upload requires a bucket destination.r7r=zAStdin upload requires a full destination path including filename.z"Cannot pipe to stdout for uploads.zNSource must be a file, not a directory. Use `hf buckets sync` for directories.zJSource path must include a file name, not just a bucket or directory path.rr>rbiHT)rk Downloaded)addUploadedstdinzSource file not found: )$rr"r copy_filesr'rtrur2rrwr endswithosrisdirrsplitrrr rdownload_bucket_filesopenreadsysstdoutbufferwritersepdirnamemakedirsrrisfilebasenamer)rrrlrmrx src_is_hf dst_is_hf src_is_bucket dst_is_bucket src_is_stdin dst_is_stdoutrzrrerifilenamepbar_was_disabledtmp_dirtmp_pathrchunk local_path parent_dir remote_pathdatas r!rrrsA4 5 ! ! !Cc""I46M#$6$6I#C((MtO<(<(!; < < $ (%''' ++--;!w||GX>>H--i68:L9MNNNh--;'(vvj'9'99e;J-33E:::()vvj'9'99e;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;)+(*****++(+(****+{% s## !s||BF';'; !s||C?P?P !W\\#x88  44J 7 J6666  % %i6:2F1G H H H J|* = = = = = = Y!3C!8!8 ;y$$&& yk/B.CDDD :7444444w~~c"" F$%Ds%D%DEE E.s33 6 R<<'**3//KK __S ! ! ! 27#3#3C#8#88KK K y[/A.BCCC :3},Wi,W,W+,W,WXXXXXXsn&A== B-!B((B-'L5A K5?AK K5K" "K5%K" &K5) L5K99L<K9=LL')F)FF)rrF)NNFFFNNFNNNFFFFN)B__doc__rrrrtypingrrthuggingface_hubrhuggingface_hub._bucketsrrrr r r r huggingface_hub.utilsr rrrr _cli_utilsrrrrrr_outputrr get_logger__name__logger buckets_clir2boolr"tupler(intfloatr4r9rrTrrIcommandautoArgumentOptionrgr}rrrrrrrrrrr#r!r-s>=  ######/.......  H % %m!EFFF $$$$$$ S U38_    "  sU{ D S    //D/$/3////!<<  \) *<< < #Y <<<<F )) ) 9) ) )  )  ) ))))X    .   4 9-,O,O g    ,O  +    ,O  F    ,O* +,O, -,O. /,O,O,O  ,O^CD     0     4 9G? ?  d W    ?    7    ? $  F    %? 2   C    3? B C? D E? F G? H I? ? ? ? D KTz K $J K K K  K : K  K K K KF;6;6;6;6 ;6 : ;6  ;6;6;6;6|  (5 4 9"" Y    " " " """",  .   4 9/2626 Y    26    ,     26  F    26, -26. /260 126262626j    6     4 9]BHBH a    BH   C    BH$    ,     %BH4  K    5BHB S D d    CBHN S D _    OBHZ [BH\ ]BH^ _BHBHBH  BHJ  * 4 9#K#K  `    #K  e     #K #K #K #K#K#K#KV       &          IzPzP  d ^     zP  d c     zP   B     zP&  N    'zP4  O    5zPB  d  P     CzPN  d  6     OzPZ  H    [zPh S D I    izPt S D I    uzP@ d  ;    AzPL  T    MzPZ  V    [zPh   8    izPx    "     yzPH IzPJ KzPzPzP  zPD     *  4 9sYsY  ^U^!p q q qq sY  d NEN(jkkkk  sY sY sY sYsYsYsYsYsYr#