
    j+                         d dl Z d dlZd dlmZ d dlZd dlZd dlm	Z	 d dl
mZ d dlmZmZ ddlmZmZmZmZ  ej        d          Ze                    ej                    G d	 d
          ZdS )    N)Path)DataType)MilvusException)CollectionSchemaFieldSchema   )DYNAMIC_FIELD_NAMEMBNUMPY_TYPE_CREATORBulkFileTypebulk_bufferc                       e Zd Zej        fdedefdZedefd            Z	de
fdZdefd	Zd
efdZde
defdZde
fdZde
fdZde
fdZdS )Bufferschema	file_typec                    i | _         i | _        || _        |j        D ]/}|j        r|j        rg | j         |j        <   || j        |j        <   0t          | j                   dk    r|                     d           |j	        r>g | j         t          <   t          t          t          j                  | j        t          <   d S d S )Nr   z/Illegal collection schema: fields list is empty)namedtype)_buffer_fields
_file_typefields
is_primaryauto_idr   len_throwenable_dynamic_fieldr	   r   r   JSON)selfr   r   fields       e/lsinfo/ai/hellotax_ai/base_platform/venv/lib/python3.11/site-packages/pymilvus/bulk_writer/buffer.py__init__zBuffer.__init__)   s    
 #] 	- 	-E EM ')DL$',DL$$t|!!KKIJJJ & 	/1DL+,/:'x}0 0 0DL+,,,	 	    returnc                     t          | j                  dk    rdS | j        D ]}t          | j        |                   c S d S )Nr   )r   r   )r   ks     r!   	row_countzBuffer.row_countA   sL    t|!!1 	( 	(At|A'''''tr#   msgc                 V    t                               |           t          |          )N)message)loggererrorr   )r   r(   s     r!   r   zBuffer._throwJ   s%    Sc****r#   xc                     t          |t          j                  r|                                S t          |t          j                  r|                                S |S )N)
isinstancenpndarraytolistgenericitem)r   r-   s     r!   _raw_objzBuffer._raw_objN   sJ    a$$ 	88::a$$ 	6688Or#   rowc                    i }t           |v r>t          |t                    t                    s|                     dt            d           |D ]w}|t           k    r|                    ||                    )|| j        vr|                     ||                   ||<   Q| j        |                             ||                    xt           | j        v r'| j        t                                        |           d S d S )NzDynamic field 'z' value should be JSON format)r	   r/   dictr   updater   r5   append)r   r6   dynamic_valuesr&   s       r!   
append_rowzBuffer.append_rowV   s    $$Z<N8OQU-V-V$KK[*<[[[\\\ 	/ 	/A&&&%%c!f---$$$(MM#a&$9$9q!!Q&&s1v....--L+,33NCCCCC .-r#   
local_pathc           	         d}| j         D ]y}|dk     rt          | j         |                   }#|t          | j         |                   k    r8t          | j         |                   }|                     d| d| d|            z| j        t          j        k    r | j        |fi |S | j        t          j        k    r | j        |fi |S | j        t          j	        k    r | j
        |fi |S |                     d| j                    g S )Nr   zColumn z row count z- doesn't equal to the first column row count zUnsupported file tpye: )r   r   r   r   r   NUMPY_persist_npyr   _persist_json_rowsPARQUET_persist_parquet)r   r=   kwargsr'   r&   buffer_k_lens         r!   persistzBuffer.persisth   s:   	 	 	A1}}Q00		c$,q/2222"4<?33rarrLrrgprr  
 ?l000$4$Z::6:::?l///*4*:@@@@@?l222(4(>>v>>>?do??@@@	r#   c                    g }t          t          t          | j                                                                      }| j        D ]u}t          |                              |dz             }|                    t          |                     	 t          |          	                    d           d }| j
        |         }|j        t          j        k    r|j        }	t          |	j                 }n*|j        j        t          v rt          |j        j                 }|j        t          j        k    rCg }
| j        |         D ])}|
                    t%          j        |                     *|
| j        |<   |j        t          j        k    r|                     d| d           |j        t          j        t          j        hv rdg }| j        |         D ]=}|                    t1          j        ||                                                     >t1          j        ||          }n!t1          j        | j        |         |          }t1          j        t          |          |           n2# t:          $ r%}|                     d| d|            Y d }~nd }~ww xY wt<                              d| d	|            wt          |          t          | j                  k    rt<                               d
           |D ]#}t          |          !                                 $t          |          "                                 |#                                 |                     d
           |S )Nz.npyT)exist_okFailed to persist file zF, error: milvus doesn't support parsing sparse vectors from numpy filer   	, error: Successfully persist file , row count: z?Some of fields were not persisted successfully, abort the files)$r   nextiterr   valuesr   joinpathr:   strmkdirr   r   r   ARRAYelement_typer   r   r   jsondumpsSPARSE_FLOAT_VECTORr   FLOAT16_VECTORBFLOAT16_VECTORr0   
frombufferr2   arraysave	Exceptionr+   infor,   unlinkrmdirclear)r   r=   rE   	file_listr'   r&   full_file_namedtfield_schemarV   str_arrvalabarrefs                    r!   rA   zBuffer._persist_npy   sb   	T$,"5"5"7"78899::	 *	_ *	_A!*--66q6zBBNS00111%TZ  &&&555 #|A%77#/#<L+L,=>BB!',0BBB+L,>,CDB  %66 G#|A 8 8tz#7777&-DLO  %)EEEKKa. a a a    %(*A8C[)\\\A!\!_ F Fq!;!;!;!B!B!D!DEEEE(1B///CC(4<?"===CN++S1111 T T TRnRRqRRSSSSSSSST KK]^]]R[]]^^^^y>>S....LLZ[[[ ! !Q    ""$$$OOKKYZZZs   GI--
J7JJc                    g }t          t          t          | j                                                                      }d}||k     ri }| j                                        D ]\  }}| j        |         }	|	j        t          j	        t          j
        hv rg|	j        t          j
        k    rt          j        d          nt          j        }
t          j        ||         |
                                          ||<   ||         ||<   |                    |           |dz   }||k     d|i}t!          |dz             }	 |                    d          5 }t%          j        ||d	           d d d            n# 1 swxY w Y   n2# t(          $ r%}|                     d
| d|            Y d }~nd }~ww xY wt,                              d| dt          |                      t1          |          gS )Nr   bfloat16rK   r   rowsz.jsonw   )indentrJ   rL   rM   rN   )r   rO   rP   r   rQ   itemsr   r   r   rZ   r[   r0   float16r\   r2   r:   r   openrW   dumpr_   r   r+   r`   rS   )r   r=   rE   rq   r'   	row_indexr6   r&   vrg   rf   data	file_path	json_filerm   s                  r!   rB   zBuffer._persist_json_rows   sS   T$,"5"5"7"78899::		)##C**,, * *1  $|A%(*A8C[)\\\ ).(2JJJ ,,,Z 
  ]1Y<rBBBIIKKCFFy\CFFKK!AI! )##& D
 g-..		K$$ 5		$	!44445 5 5 5 5 5 5 5 5 5 5 5 5 5 5 	K 	K 	KKKI)IIaIIJJJJJJJJ	K 	TTTTTTUUUIs<   F E:.F :E>>F E>F 
F5F00F5c           
      H   t          |dz             }i }| j        D ]}| j        |         }|j        t          j        t          j        hv rSg }| j        |         D ])}|                    t          j	        |                     *t          j        |d           ||<   |j        t          j        t          j        hv rig }	| j        |         D ]@}|	                    t          j        |t           |j        j                                      At          j        |	          ||<   
|j        t          j        t          j        hv rig }	| j        |         D ]@}|	                    t          j        |t           |j        j                                      At          j        |	          ||<   |j        t          j        k    rkt           |j        j                 }
g }	| j        |         D ]+}|	                    t          j        ||
                     ,t          j        |	          ||<   |j        j        t           v r=t           |j        j                 }
t          j        | j        |         |
          ||<   bt          j        | j        |                   ||<   d}d}d}d|v rd|v r|                    dd	t0          z            }|                    dd
          }|                    dd
          }t3          ||z            d
z   }t3          ||z            }t5          ||          }t7          ||          }t          j        |          }|                    ||d           t<                              d| d| d| d|            tA          |          gS )Nz.parquetrK   i  i'  i@B buffer_sizebuffer_row_countrow_group_bytes    r   )r{   pyarrow)row_group_sizeenginerM   z, total size: rN   z, row group size: )!r   r   r   r   r   r   rY   r:   rW   rX   pdSeriesBINARY_VECTORFLOAT_VECTORr0   r]   r   r   rZ   r[   r\   rU   rV   getr
   intmaxmin	DataFrame
to_parquetr+   r`   rS   )r   r=   rE   r|   r{   r&   rg   rh   ri   rl   rf   row_group_size_minr   row_group_size_maxr   r   r   size_per_row
data_frames                      r!   rD   zBuffer._persist_parquet   s   j011	 !	5 !	5A<?L!hmX5Q%RRR<? 4 4CNN4:c??3333)G4888Q#(>@U'VVV<? a aCJJrx3ElFXF]3^___````)C..Q#(?AY'ZZZ <?  CJJc1CLDVD[1\]]]    )C..Q#x~55'(A(FG<? 8 8CJJrx26667777)C..Q#(,>>>'(:(?@)DLO2>>>Q )DLO44Q "$F""'9V'C'C$jj!27 O !**]A66K%zz*<a@@{-==>>BL <!?@@N 1CDDN 1CDDN \t,,,
nY 	 	
 	
 	
 	P P P+ P P+P P?MP P	
 	
 	
 Ir#   N)__name__
__module____qualname__r   r@   r   r"   propertyr   r'   rS   r   objectr5   r8   r<   listrG   rA   rB   rD    r#   r!   r   r   (   s@        #/"4       0 3    X+# + + + +&    Dd D D D D$# D    .7s 7 7 7 7r! S !  !  !  ! F@ 3 @  @  @  @  @  @ r#   r   )rW   loggingpathlibr   numpyr0   pandasr   pymilvus.client.typesr   pymilvus.exceptionsr   pymilvus.orm.schemar   r   	constantsr	   r
   r   r   	getLoggerr+   setLevelDEBUGr   r   r#   r!   <module>r      s=                        0 / / / / /       
            
	=	)	)    s  s  s  s  s  s  s  s  s  s r#   