yjMUddlmZddlZddlmZmZmZddlZddl Z ddl m Z ddl m Z ddl m Z mZdd lmZdd lmZmZmZmZmZmZmZer/dd lmZdd lmZmZdd lmZee ee!dfZ"de#d<gZ$iej%j&j'dej%j&j(dej%j&j)dej%j&j*dej%j&j+dej%j&j,dej%j&j-dej%j&j.dej%j&j/dej%j&j0dej%j&j1dej%j&j2dej%j&j3dej%j&j4dej%j&j5dej%j&j6d Z7d>d"Z8d#Z9d?d(Z: d@d*Z;d@d+Z< d@d,Z=e>e!ee fe?ee fd-fdAd7Z@Gd8d9ZAGd:d;ZBGd<d=ZCdS)B) annotationsN) TYPE_CHECKINGAnyUnion)pir)Value)_PADDLE_PIR_DTYPE_2_NUMPY_DTYPE ParameterMeta)core)EagerParamBaseVariable_cpu_num _cuda_idsdefault_main_programin_dygraph_mode in_pir_mode) TypeAlias) DTypeLike ShapeLike) _DTypeLiteral) _ClassInfo.rrbool float8_e4m3fn float8_e5m2float16uint16float32float64int8int16int32int64uint8 complex64 complex128pstringrawNCHWcb|jdkr|tjS|dkrtj|gd}tjdtjg|j}tj||j}|dkrtj|gd}|S)NrNHWC)rr rcdtjdtjd|ddz S)Nz)convert_float_to_uint16..Ls(&-fk$&:&:;;A>"Dotypes)rrr-r ) sizeviewnpr transpose vectorizeflatreshapeshape)data data_formatnew_datas r8convert_float_to_uint16rHDs yA~~yy###f|D,,,//r|DD { iHz(DJ//Hf<,,,77 Or:ctjdtjg|j}tj||jS)Ncdtjdtjd|dzdS)Nr0r/r1rr2r6s r8r9z)convert_uint16_to_float..Xs'&-fk$R&@&@AA!Dr:r;)r?rArrBrCrD)rErGs r8convert_uint16_to_floatrKVsPr|DD | iH :h + ++r:dtyperreturnrct|tjjr|tvr t|St|tjr|t vr t |Snt|tr|ttj tj tj tj tjtjtjtjtjtjtjtjtjfvr|jSn|dvrt1|S|dvrdSt3d|)N)rrruint32uint64rr int4r!r"r#r$r%r&r'rr)bfloat16rzdtype must be any of [bool, float16, uint16, float32, float64, int8, int16, int32, int64, uint8, complex64, complex128, bfloat16], but received ) isinstancer VarDescVarType_PADDLE_DTYPE_2_NUMPY_DTYPEDataTyper typerr?rrrOrPrr r!r"r#r$r%r&r'__name__str TypeErrorrLs r8 convert_dtyper]^s2%-..6 / / /.u5 5%''3 3 3 3259 9 4 E4 0   J I I I J J G H H H H L M   > !! &    ,u::  L 8  WOT W W  r:c(trLddlm}|jrtt t f}ntt f}t|||||nt||ttf||t|j ||||dS)Nrlazy_init_helper) rnn.initializer.lazy_initrastater r r check_typer check_dtyperL)input input_nameexpected_dtypeop_name extra_messageraexpecteds r8check_variable_and_dtyperls}} Q??????     # .}n=HH}-H5*hGGGG5*x&7-PPP Z-PPPPPr:c trdSddlm}ddlm}|r.t |t s|f}|tjj fz }nDt |tjj r%|j std|d|dt ||s+td|d|d |d t|d | dS) Nrr`r )in_to_static_modezPlease use `with base.dygraph.guard()` as context or `paddle.disable_static()` to switch to dygraph mode firstly. Because received '' in z is an Eager Tensor.z The type of ' must be , but received . ) rrbra dygraph.basernrStupler eagerTensorrcr[rX)rfrg expected_typerirjrarns r8rdrdsK <;;;;;////// -// -*,M$*+-- E4:, - - 6F6F6H6H6N  P!+ P P29 P P P    e] + +  {J { {W { {} { {]abg]h]h { {ly { {     r:c trdSt||vr+td|d|d|dt|d| dS)NzThe data type of 'rorprqrr)rr]r[) input_dtypergrhrirjs r8reres[!!77 Q Q Q' Q QN Q Qcpq|c}c} Q QBO Q Q   87r:)r#r$rDrrirZexpected_shape_typeexpected_element_typeexpected_tensor_dtypetuple[_DTypeLiteral, ...]Nonec trdSt|d|||t|ttfss|D]p}t|d|||Zt|ttfr>t |jd||dd|q|5t|ttfrt |jd||dSdSdS)NrDzelement of shapez9If element of shape is Tensor, its data type should be {}z, ) rrdrSrr rerLformatjoin)rDrirzr{r|items r8 check_shapers%"ug2G<<<( % 22(  D t/1F P P P$0Zx'660J&)OVV "788 (Z % ..( EK*?IIIII)(((r:c2eZdZdZdZdZdZdZdZdS)DataToDenseTensorConverterc||_||_||_d}|jD]}|dkr|dz }|dkr d|_nt||_|dSNrr )place lod_levelrDr]rL_reset)selfrrrDrLnegative_countss r8__init__z#DataToDenseTensorConverter.__init__s~ "   A1uu!#!!! "#5))  r:cZg|_dt|jD|_dS)Ncg|]}gSr).0_s r8 z5DataToDenseTensorConverter._reset..s6661B666r:)rErangerlodrs r8rz!DataToDenseTensorConverter._resets, 66dn 5 5666r:cH|||j|jdSN) _feed_impl_rr)rrEs r8feedzDataToDenseTensorConverter.feeds$ tx88888r:c|dkr|j|dS|dt||D]$}|||dd|dz %dSr)rEappendlenr)rrErr each_datas r8rz&DataToDenseTensorConverter._feed_impl_s >> I  T " " " " " FMM#d)) $ $ $! D D   CGY]CCCC D Dr:ct|j|D]1\}}||kr&|dkr |dkrtd|jd|2dS)Nrz2Shape not match. What is defined in data layer is , but receive )ziprD ValueError)rrDs1s2s r8 _check_shapez'DataToDenseTensorConverter._check_shape$sm$*e,,  FBRxxB!GGa jjjchjj  r:ctj|j|j}|jrst |jt |jkrI ||j}n-#t$r td|jd|jwxYwtj }| ||j |j dkr| |j||S)Nr\z0Reshape error. What is defined in data layer is rr)r?arrayrErLrDrrCrr DenseTensorsetrrset_recursive_sequence_lengthsrr)rarrts r8donezDataToDenseTensorConverter.done+shty 333 : 39~~TZ00++dj11CC!$p4:ppehenpp     c4: >A   , ,TX 6 6 6 s A..*BN) rY __module__ __qualname__rrrrrrrr:r8rrsq   777999DDDr:rc eZdZdZdZdZdS)BatchedTensorProviderc ||_||_||_g|_||_|D]_}t s|jdks Jd|jt|jd|j |j `dS)Nrzlod_level must be 0rrrDrL) r batch_size generator converters drop_lastrrrrrDrL)r feed_listrrrrvars r8rzBatchedTensorProvider.__init__>s $""  C== A})))+@))) O " "**))       r:c$d|jDS)Nc6g|]}|Sr)rrcs r8rz/BatchedTensorProvider._done..Rs 222Q222r:)rrs r8_donezBatchedTensorProvider._doneQs22$/2222r:c#bKd}|D]_}t||jD]\}}|j| |dz }||jkrd}|V`|js|dkr|VdSd|jDdS)Nrr c6g|]}|Sr)rrs r8rz2BatchedTensorProvider.__call__..bs 1 1 1AQXXZZ 1 1 1r:)rrrrErrrr)ridx each_sample each_sloteach_converters r8__call__zBatchedTensorProvider.__call__Ts>>++ # #K-0do-N-N 6 6) >#**95555 1HCdo%%jjll"""~ 2#''**,,      1 1 1 1 1 1 1 1r:N)rYrrrrrrr:r8rr=sA&33322222r:rc&eZdZdZddZdZdZdS) DataFeedera :api_attr: Static Graph DataFeeder converts the data that returned by a reader into a data structure that can feed into Executor. The reader is usually a python generator that returns a list of mini-batch data entries. Parameters: feed_list (list): Variables or names of Variables that need to feed. place (:ref:`api_paddle_CPUPlace` | :ref:`api_paddle_CUDAPlace` ): place indicates the device (CPU | GPU) the data will be fed into, if you want to feed data into GPU, please using :code:`base.CUDAPlace(i)` (:code:`i` represents the GPU id), or if you want to feed data into CPU, please using :code:`base.CPUPlace()`. program (:ref:`api_paddle_static_Program` , optional): The Program that will feed data into, if program is None, it will use default_main_program(). Default None. Raises: :code:`ValueError` - If some Variables are not in this Program. Example: .. code-block:: python >>> import numpy as np >>> import paddle >>> from paddle import base >>> paddle.enable_static() >>> place = paddle.CPUPlace() >>> def reader(): ... for _ in range(4): ... yield np.random.random([4]).astype('float32'), np.random.random([3]).astype('float32'), ... >>> main_program = paddle.static.Program() >>> startup_program = paddle.static.Program() >>> with paddle.static.program_guard(main_program, startup_program): ... data_1 = paddle.static.data(name='data_1', shape=[None, 2, 2], dtype='float32') ... data_2 = paddle.static.data(name='data_2', shape=[None, 1, 3], dtype='float32') ... out = paddle.static.nn.fc(x=[data_1, data_2], size=2) ... # ... >>> feeder = base.DataFeeder([data_1, data_2], place) >>> exe = paddle.static.Executor(place) >>> exe.run(startup_program) >>> feed_data = feeder.feed(reader()) >>> # print feed_data to view feed results >>> # print(feed_data['data_1']) >>> # print(feed_data['data_2']) >>> outs = exe.run( ... program=main_program, ... feed=feed_data, ... fetch_list=[out] ... ) >>> print(outs) Nc&g|_g|_g|_g|_||_t r|t j}|D]}t|trtdt|tstd|j|j|j|j|jd|j|jdS|t}|D]}t|tr(|d|}t|t(tfstd|j|j|j|j|j|j|j|jdS)Nz+In PIR Mode, Not supported string input yetz(Feed list should contain a list of Valuerz+Feed list should contain a list of variable) feed_dtypes feed_names feed_shapesfeed_lod_levelrrrr rrSrZrr r[rrLnamerDblockrrr)rrrprogrameach_vars r8rzDataFeeder.__init__s  == 8(7799% 8 8h,,$E"(E22P#$NOOO ''777&&x}555#**1--- ''7777 8 8.00% 8 8h,,>&}}Q//33H==H!(Xu,=>>#E ''777&&x}555#**8+=>>> ''7777 8 8r:c 2 g}t|j|j|jD]1\}}}|t |j|||2 fd tj rq|D]m}t|t|ks)Jdt|t|fzt||D]\}} ||nny|D]v}t|t|ks)Jdt|t|fzt||D]\}}| |wi} t|j |D]\} }| | | <| S)au According to :code:`feed_list` of :code:`DataFeeder` and :code:`iterable` , converts the input into a data structure that can feed into Executor. Parameters: iterable (generator): user defined python generator to read the raw input data Returns: :code:`dict`: a :code:`dict` that contains (variable name - converted tensor) pairs Example: .. code-block:: python >>> # In this example, reader - generator will return a list of ndarray of 3 elements >>> # feed API will convert each ndarray input into a tensor >>> # the return result is a dict with keys: data_1, data_2, data_3 >>> # result['data_1'] a LoD-Tensor with shape of [5, 2, 1, 3]. 5 is batch size, and [2, 1, 3] is the real shape of data_1. >>> # result['data_2'], result['data_3'] are similar. >>> import numpy as np >>> import paddle >>> from paddle import base >>> paddle.enable_static() >>> def reader(limit=5): ... for i in range(1, limit + 1): ... yield np.ones([6]).astype('float32') * i , np.ones([1]).astype('int64') * i, np.random.random([9]).astype('float32') ... >>> data_1 = paddle.static.data(name='data_1', shape=[None, 2, 1, 3]) >>> data_2 = paddle.static.data(name='data_2', shape=[None, 1], dtype='int64') >>> data_3 = paddle.static.data(name='data_3', shape=[None, 3, 3], dtype='float32') >>> feeder = base.DataFeeder(['data_1','data_2', 'data_3'], paddle.CPUPlace()) >>> result = feeder.feed(reader()) >>> print(result['data_1']) >>> print(result['data_2']) >>> print(result['data_3']) rct|ttfr|D]}||dS||dSr)rSlistrtr) converterrEr feed_datas r8rz"DataFeeder.feed..feed_datasa$u .. % //DIi....//t$$$$$r:zDThe number of fields in data (%d) does not match len(feed_list) (%d))rrrrrrrpaddle framework use_pir_apirrrr) riterablerrrDrLrrrret_dict each_namers @r8rzDataFeeder.feedsP '*  !143C( (   #Iue   **'      % % % % %   ' ' ) ) 3' 9 9 ;''3y>>999,%%s9~~6<799925Y 1L1L99-NIIni88889  9 ( 3 3 ;''3y>>999,%%s9~~6<799925Y 1L1L33-NI"'' 22223),T_i)H)H 8 8 %I~"0"5"5"7"7HY  r:c|t|St|jtjrt t StSr)intrSrr CUDAPlacerrr)r num_placess r8_get_number_of_places_z!DataFeeder._get_number_of_places_sF  !z?? "  DN 3 3 y{{## #:: r:r)rYrr__doc__rrrrr:r8rresV==~!8!8!8!8FQQQfr:r)r*)rLrrMr)r^) rDrrirZrzrr{rr|r}rMr~)D __future__rr3typingrrrnumpyr?rrr pir.corer r r^r rrrrrrrrtyping_extensionsrpaddle._typingrrpaddle._typing.dtype_likerrXrtr__annotations____all__rTrUBOOL FP8_E4M3FNFP8_E5M2FP16BF16FP32FP64INT8INT16INT32INT64UINT8 COMPLEX64 COMPLEX128STRINGRAWrVrHrKr]rlrdrerrrrrrrr:r8rs#"""""" ,,,,,,,,,, EEEEEEEEG++++++33333333777777!$s)U3D-E"EFJFFFF LvL#_ L!= Ly   Lx   Ly  Ly Lv L L L L L"K L#\ L  Le!($,,,<<<<@?A Q Q Q Q     DEG     "   '  ) 8J(J(J(J(J(JV44444444n%2%2%2%2%2%2%2%2P||||||||||r: