o
    z3iu                     @   s   d dl mZmZ d dlZerddlmZ dd Zdd Zd	ed
e	defddZ
d!dedee	 fddZde	de	fddZdee	 d
e	ddfddZdedee	 defddZdddee	 fddZdddee	 ddfdd ZdS )"    )TYPE_CHECKINGAnyN   )FeatureTypec                  O   >   zt jjj| i |W S  ty   t jjj| i | Y S w N)pdiojsonujson_dumpsAttributeErrordumpsargskwargs r   a/lsinfo/ai/hellotax_ai/llm_service/venv_embed/lib/python3.10/site-packages/datasets/utils/json.pyr   
   
   r   c                  O   r   r   )r   r	   r
   ujson_loadsr   loadsr   r   r   r   r      r   r   examplejson_field_pathreturnc                    sx    r' ^} | d u rd S |dkr fdd| D S i | |t | | iS zt|  W | S  ty;   t|  Y S w )Nr   c                       g | ]}t | qS r   json_encode_field.0xr   r   r   
<listcomp>        z%json_encode_field.<locals>.<listcomp>)r   getr   	Exceptionr   )r   r   fieldr   r   r   r      s   
r   Fexamplesc                    s  g }dd | D } | sg S g | fg}|r| d\} tdd  D r^|s(|r=t fdd D s7 d s=|| nMdd  D D ]fd	d D } | sRqD||g | f qDn,td
d  D r|dd  D } | sqq||dg | f ntdd  D r|| |s|S )Nc                 S   s   g | ]}|d ur|qS r   r   r   r   r   r   r   r    /   s    z7find_mixed_struct_types_field_paths.<locals>.<listcomp>r   c                 s       | ]}t |tV  qd S r   )
isinstancedictr   r   r   r   	<genexpr>5       z6find_mixed_struct_types_field_paths.<locals>.<genexpr>c                 3   s$    | ]}t |t  d  kV  qdS )r   N)setr   )contentr   r   r*   6   s   " c                 S   s   h | ]	}|D ]}|qqS r   r   )r   r   r$   r   r   r   	<setcomp>9   s    z6find_mixed_struct_types_field_paths.<locals>.<setcomp>c                    s(   g | ]} |v r|  d ur|  qS r   r   r   )subfieldr   r   r    :   s   ( c                 s   r'   r   )r(   listr   r   r   r   r*   >   r+   c                 S   s"   g | ]}|D ]}|d ur|qqS r   r   )r   sublistr   r   r   r   r    ?   s   " c                 s   s    | ]
}t |ttfV  qd S r   )r(   r)   r0   r   r   r   r   r*   C   s    )popallanyappend)r%   
allow_rootmixed_struct_types_field_pathspaths_and_content_to_checkpathr   )r-   r/   r   #find_mixed_struct_types_field_paths-   s2   
&
r:   err_strc                 C   s:   |  ddd ddd d}dd | dD }|S )NzColumn(   z) changed fromr   /c                 S   s   g | ]
}|d kr
dn|qS )z[]r   r   )r   segr   r   r   r    L   s    z?get_json_field_path_from_pyarrow_json_error.<locals>.<listcomp>)splitrsplitstrip)r;   json_field_path_strr   r   r   r   +get_json_field_path_from_pyarrow_json_errorH   s   "rC   json_field_pathsc                 C   sF   t t| D ]}| | d t| |kr|| |<  d S q| | d S r   )rangelenr5   )rD   r   ir   r   r   insert_json_field_pathP   s   rH   original_batchc                    sJ   dd |   D }|D ]  fdd|D }qddd |D  }|S )Nc                 S      g | ]}t |qS r   )r   )r   liner   r   r   r    [       z4json_encode_fields_in_json_lines.<locals>.<listcomp>c                    r   r   r   r&   r   r   r   r    ]   r!   
c                 S   rJ   r   )r   r&   r   r   r   r    ^   rL   )
splitlinesjoinencode)rI   rD   r%   batchr   r   r    json_encode_fields_in_json_linesZ   s
   rR   featurer   c                    s0   ddl m m} g  fdd}|| | S )Nr   Json_visit_with_pathc                    s   t |  r
| | S r   )r(   r5   )_featurefeature_pathrU   rD   r   r   get_json_type_pathg   s   

z=get_json_field_paths_from_feature.<locals>.get_json_type_pathdatasets.features.featuresrU   rV   )rS   rV   rZ   r   rY   r   !get_json_field_paths_from_featureb   s
   
r]   c                    s,   ddl m m}  fdd}|| |} | S )Nr   rT   c                    s   |v r  S | S r   r   )rS   rX   rY   r   r   set_json_types   s   z0set_json_types_in_feature.<locals>.set_json_typer[   )rS   rD   rV   r^   r   rY   r   set_json_types_in_featurep   s   
r_   )F)typingr   r   pandasr   features.featuresr   r   r   strr   r0   r:   rC   rH   bytesrR   r]   r_   r   r   r   r   <module>   s    
