o ~ri'@sdZddlZddlZddlmZmZmZddlZddlm Z ddl m Z m Z m Z ddlmZmZddlmZmZmZmZmZmZmZmZmZmZmZmZmZmZe eeZ!eeZ"ej#d d d e"De$d Z%eee$ej&d d'e!dee!dfZ(eddZ)e)j*dgddddddddej+ddf dedededeee%ej&ddfdede(ded ed!ed"dfd#d$Z,e)j*d%d&d'gd   d9sr)typezComma-separated properties to return. When used, only the listed properties (and id) are returned. Example: '--expand=downloads,likes,tags'. Valid: z, .)helpcallbackz"Interact with datasets on the Hub.)r"z list | ls)zhf datasets lsz*hf datasets ls --sort downloads --limit 10zhf datasets ls --search "code")examples Fsearchauthorfiltersortz Sort results.limitexpandformatquiettokenreturnc CsJt|d} |r |jnd} dd| j|||| ||dD} t| ||ddS)zList datasets on the Hub.r.NcSsg|]}t|qSr)r)r dataset_inforrr aszdatasets_ls..)r(r'r&r)r*r+r,r-)rvalue list_datasetsr) r&r'r(r)r*r+r,r-r.apisort_keyresultsrrr datasets_lsHs  r9infoz&hf datasets info HuggingFaceFW/finewebz9hf datasets info my-dataset --expand downloads,likes,tags dataset_idz+The dataset ID (e.g. `username/repo-name`).revisionc Cst|d}z |j|||d}Wn,ty$}z td|d|d}~wty;}z td|d|d|d}~wwttjt|d d dS) z>Get info about a dataset on the Hub. Output is in JSON format.r0)repo_idr<r+z Dataset 'z ' not found.Nz Revision 'z' not found on 'z'.)indent) rr1rrrprintjsondumpsr)r;r<r+r.r6r:errr datasets_infoos rDparquet)z(hf datasets parquet cfahlgren1/hub-statsz8hf datasets parquet cfahlgren1/hub-stats --subset modelsz6hf datasets parquet cfahlgren1/hub-stats --split trainz6hf datasets parquet cfahlgren1/hub-stats --format jsonsubsetz--subsetz(Filter parquet entries by subset/config.splitz Filter parquet entries by split.c sLt|d}|j||d}fdd|D}dd|D} t| ||dddS) z/List parquet file URLs available for a dataset.r0)r=configcs"g|] }dus |jkr|qS)NrGrentryrIrrr2s"z$datasets_parquet..cSs"g|] }|j|j|j|jdqS))rFrGurlsize)rHrGrLrMrJrrrr2srL)r,r-id_keyN)rlist_dataset_parquet_filesr) r;rFrGr,r-r.r6entriesfilteredr8rrIrdatasets_parquets rRsqlzhf datasets sql "SELECT COUNT(*) AS rows FROM read_parquet('https://huggingface.co/api/datasets/cfahlgren1/hub-stats/parquet/models/train/0.parquet')"zhf datasets sql "SELECT * FROM read_parquet('https://huggingface.co/api/datasets/cfahlgren1/hub-stats/parquet/models/train/0.parquet') LIMIT 5" --format jsonzRaw SQL query to execute.c CsJzt||d}Wnty}ztt||d}~wwt||dddS)zAExecute a raw SQL query with DuckDB against dataset parquet URLs.) sql_queryr.NFr3)r ImportErrorrstrr)rSr,r.resultrCrrr datasets_sqls rX)NNN)1__doc__enumrAtypingrrrtyperhuggingface_hub._dataset_viewerrhuggingface_hub.errorsrrrhuggingface_hub.hf_apir r _cli_utilsr r rrrrrrrrrrrrsorted_EXPAND_PROPERTIES _SORT_OPTIONSEnumrVrOptionjoin ExpandOpt datasets_clicommandtabler9ArgumentrDrRrXrrrrs   @