|jHPUddlmZddlmZmZmZmZddlmZer%ddl m Z ddl m Z ddl mZddlmZed Zd ed <dd lZdd lmZdd lZddlmZddlmZgZd dZd!dZGddeedefZ d Z!dZ"dZ#dZ$Gddee%dZ&d S)") annotations) TYPE_CHECKINGAnyCallableLiteral) TypeAlias)Sequence) _DTypeLiteral) _Transform)_ImageDataType) z.jpgz.jpegz.pngz.ppmz.bmpz.pgmz.tifz.tiffz.webpr_AllowedExtensionsN)Image)Dataset) try_importfilenamestr extensions Sequence[str]returnboolct|ttfs Jdtd|D}||S)zChecks if a file is a valid extension. Args: filename (str): path to a file extensions (list[str]|tuple[str]): extensions to consider Returns: bool: True if the filename ends with one of given extensions z#`extensions` must be list or tuple.c6g|]}|S)lower).0xs m/lsinfo/ai/hellotax_ai/data_center/backend/venv/lib/python3.11/site-packages/paddle/vision/datasets/folder.py z'has_valid_extension..As 666a 666) isinstancelisttuplerendswith)rrs rhas_valid_extensionr%4sk j4- 0 0- 066:66677J >>   $ $Z 0 00r c,g}tj|}fd}t|D]}tj||}tj|sBttj|dD]b\}}} t| D]L} tj|| } || r| ||f} || Mc|S)Nc$t|SNr%rrs r is_valid_filez#make_dataset..is_valid_fileKs&q*55 5r T followlinks) ospath expandusersortedkeysjoinisdirwalkappend) dir class_to_idxrr+imagestargetdroot_fnamesfnamer/items ` r make_datasetrAEs* F '  S ! !C 6 6 6 6 6**,,-- ( ( GLLf % %w}}Q  %bgaT&B&B&BCC ( (OD!V ( (w||D%00 =&&( ,v"67DMM$'''  ( ( Mr ceZdZUdZded<ded<ded<ded <d ed <d ed <ded<ded< d!d"dZd#dZd$dZd ZdS)% DatasetFoldera"A generic data loader where the samples are arranged in this way: .. code-block:: text root/class_a/1.ext root/class_a/2.ext root/class_a/3.ext root/class_b/123.ext root/class_b/456.ext root/class_b/789.ext Args: root (str): Root directory path. loader (Callable|None, optional): A function to load a sample given its path. Default: None. extensions (list[str]|tuple[str]|None, optional): A list of allowed extensions. Both :attr:`extensions` and :attr:`is_valid_file` should not be passed. If this value is not set, the default is to use ('.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', '.tiff', '.webp'). Default: None. transform (Callable|None, optional): A function/transform that takes in a sample and returns a transformed version. Default: None. is_valid_file (Callable|None, optional): A function that takes path of a file and check if the file is a valid file. Both :attr:`extensions` and :attr:`is_valid_file` should not be passed. Default: None. Returns: :ref:`api_paddle_io_Dataset`. An instance of DatasetFolder. Attributes: classes (list[str]): List of the class names. class_to_idx (dict[str, int]): Dict with items (class_name, class_index). samples (list[tuple[str, int]]): List of (sample_path, class_index) tuples. targets (list[int]): The class_index value for each image in the dataset. Example: .. code-block:: python >>> import shutil >>> import tempfile >>> import cv2 >>> import numpy as np >>> import paddle.vision.transforms as T >>> from pathlib import Path >>> from paddle.vision.datasets import DatasetFolder >>> def make_fake_file(img_path: str): ... if img_path.endswith((".jpg", ".png", ".jpeg")): ... fake_img = np.random.randint(0, 256, (32, 32, 3), dtype=np.uint8) ... cv2.imwrite(img_path, fake_img) ... elif img_path.endswith(".txt"): ... with open(img_path, "w") as f: ... f.write("This is a fake file.") >>> def make_directory(root, directory_hierarchy, file_maker=make_fake_file): ... root = Path(root) ... root.mkdir(parents=True, exist_ok=True) ... for subpath in directory_hierarchy: ... if isinstance(subpath, str): ... filepath = root / subpath ... file_maker(str(filepath)) ... else: ... dirname = list(subpath.keys())[0] ... make_directory(root / dirname, subpath[dirname]) >>> directory_hierarchy = [ ... {"class_0": [ ... "abc.jpg", ... "def.png"]}, ... {"class_1": [ ... "ghi.jpeg", ... "jkl.png", ... {"mno": [ ... "pqr.jpeg", ... "stu.jpg"]}]}, ... "this_will_be_ignored.txt", ... ] >>> # You can replace this with any directory to explore the structure >>> # of generated data. e.g. fake_data_dir = "./temp_dir" >>> fake_data_dir = tempfile.mkdtemp() >>> make_directory(fake_data_dir, directory_hierarchy) >>> data_folder_1 = DatasetFolder(fake_data_dir) >>> print(data_folder_1.classes) ['class_0', 'class_1'] >>> print(data_folder_1.class_to_idx) {'class_0': 0, 'class_1': 1} >>> print(data_folder_1.samples) >>> # doctest: +SKIP(it's different with windows) [('./temp_dir/class_0/abc.jpg', 0), ('./temp_dir/class_0/def.png', 0), ('./temp_dir/class_1/ghi.jpeg', 1), ('./temp_dir/class_1/jkl.png', 1), ('./temp_dir/class_1/mno/pqr.jpeg', 1), ('./temp_dir/class_1/mno/stu.jpg', 1)] >>> # doctest: -SKIP >>> print(data_folder_1.targets) [0, 0, 1, 1, 1, 1] >>> print(len(data_folder_1)) 6 >>> for i in range(len(data_folder_1)): ... img, label = data_folder_1[i] ... # do something with img and label ... print(type(img), img.size, label) ... # (32, 32) 0 >>> transform = T.Compose( ... [ ... T.Resize(64), ... T.ToTensor(), ... T.Normalize( ... mean=[0.5, 0.5, 0.5], ... std=[0.5, 0.5, 0.5], ... to_rgb=True, ... ), ... ] ... ) >>> data_folder_2 = DatasetFolder( ... fake_data_dir, ... loader=lambda x: cv2.imread(x), # load image with OpenCV ... extensions=(".jpg",), # only load *.jpg files ... transform=transform, # apply transform to every image ... ) >>> print([img_path for img_path, label in data_folder_2.samples]) >>> # doctest: +SKIP(it's different with windows) ['./temp_dir/class_0/abc.jpg', './temp_dir/class_1/mno/stu.jpg'] >>> # doctest: -SKIP >>> print(len(data_folder_2)) 2 >>> for img, label in iter(data_folder_2): ... # do something with img and label ... print(type(img), img.shape, label) # type: ignore ... # [3, 64, 64] 0 >>> shutil.rmtree(fake_data_dir) $Callable[..., _ImageDataType] | Noneloader#Sequence[_AllowedExtensions] | Noner_Transform[Any, Any] | None transform list[str]classeszdict[str, int]r8zlist[tuple[str, int]]samplestargetsr dtypeNr<rr+_ImageDataType | NonerNonec||_||_|t}||j\}}t |j|||}t |dkr0t d|jzdzd|z|tn||_ ||_ ||_ ||_ ||_ d|D|_tj|_dS)Nrz&Found 0 directories in subfolders of:  Supported extensions are: ,cg|] }|d S)r)rss rrz*DatasetFolder.__init__..s...!...r )r<rHIMG_EXTENSIONS _find_classesrAlen RuntimeErrorr3default_loaderrErrJr8rKrLpaddleget_default_dtyperM) selfr<rErrHr+rJr8rKs r__init__zDatasetFolder.__init__s "  'J $ 2 249 = = I|Z   w<<1  .s'AAAaahhjjA16AAAr c"i|] }|| Srr)rirJs r z/DatasetFolder._find_classes..!sCCC! ACCCr )r.scandirsortrangerX)r]r7r8rJs @rrWzDatasetFolder._find_classess`BA2:c??AAA CCCCuS\\/B/BCCC  $$r indexinttuple[_ImageDataType, int]c|j|\}}||}|j||}||fS)z Args: index (int): Index Returns: tuple: (sample, target) where target is class_index of the target class. rKrErH)r]rjr/r:samples r __getitem__zDatasetFolder.__getitem__$sH|E* fT"" > %^^F++Fv~r c*t|jSr(rXrKr]s r__len__zDatasetFolder.__len__34<   r NNNN r<rrErDrrFrHrGr+rNrrO)r7rrr_)rjrkrrl) __name__ __module__ __qualname____doc____annotations__r^rWrprtrr rrCrC\sIIV10003333****    """" 8<:>15/3 0 0 0 0 0D%%%%"    !!!!!r rCr ct|d5}tj|}|dcdddS#1swxYwYdS)NrbRGB)openrconvert)r/fimgs r pil_loaderrDs dD  "Qjmm{{5!!""""""""""""""""""s)AA A c|td}||||jS)Ncv2)rcvtColorimread COLOR_BGR2RGB)r/rs r cv2_loaderrJs2 U  C << 4((#*; < <>> import shutil >>> import tempfile >>> import cv2 >>> import numpy as np >>> import paddle.vision.transforms as T >>> from pathlib import Path >>> from paddle.vision.datasets import ImageFolder >>> def make_fake_file(img_path: str): ... if img_path.endswith((".jpg", ".png", ".jpeg")): ... fake_img = np.random.randint(0, 256, (32, 32, 3), dtype=np.uint8) ... cv2.imwrite(img_path, fake_img) ... elif img_path.endswith(".txt"): ... with open(img_path, "w") as f: ... f.write("This is a fake file.") >>> def make_directory(root, directory_hierarchy, file_maker=make_fake_file): ... root = Path(root) ... root.mkdir(parents=True, exist_ok=True) ... for subpath in directory_hierarchy: ... if isinstance(subpath, str): ... filepath = root / subpath ... file_maker(str(filepath)) ... else: ... dirname = list(subpath.keys())[0] ... make_directory(root / dirname, subpath[dirname]) >>> directory_hierarchy = [ ... "abc.jpg", ... "def.png", ... {"ghi": [ ... "jkl.jpeg", ... {"mno": [ ... "pqr.jpg"]}]}, ... "this_will_be_ignored.txt", ... ] >>> # You can replace this with any directory to explore the structure >>> # of generated data. e.g. fake_data_dir = "./temp_dir" >>> fake_data_dir = tempfile.mkdtemp() >>> make_directory(fake_data_dir, directory_hierarchy) >>> image_folder_1 = ImageFolder(fake_data_dir) >>> print(image_folder_1.samples) >>> # doctest: +SKIP(it's different with windows) ['./temp_dir/abc.jpg', './temp_dir/def.png', './temp_dir/ghi/jkl.jpeg', './temp_dir/ghi/mno/pqr.jpg'] >>> # doctest: -SKIP >>> print(len(image_folder_1)) 4 >>> for i in range(len(image_folder_1)): ... (img,) = image_folder_1[i] ... # do something with img ... print(type(img), img.size) ... # (32, 32) >>> transform = T.Compose( ... [ ... T.Resize(64), ... T.ToTensor(), ... T.Normalize( ... mean=[0.5, 0.5, 0.5], ... std=[0.5, 0.5, 0.5], ... to_rgb=True, ... ), ... ] ... ) >>> image_folder_2 = ImageFolder( ... fake_data_dir, ... loader=lambda x: cv2.imread(x), # load image with OpenCV ... extensions=(".jpg",), # only load *.jpg files ... transform=transform, # apply transform to every image ... ) >>> print(image_folder_2.samples) >>> # doctest: +SKIP(it's different with windows) ['./temp_dir/abc.jpg', './temp_dir/ghi/mno/pqr.jpg'] >>> # doctest: -SKIP >>> print(len(image_folder_2)) 2 >>> for (img,) in iter(image_folder_2): ... # do something with img ... print(type(img), img.shape) # type: ignore ... # [3, 64, 64] >>> shutil.rmtree(fake_data_dir) rDrErFrrIrKrGrHNr<rr+rNrrOc@||_tg}tj|}fd}t tj|dD]X\}}} t | D]B} tj|| } || r|| CYt|dkr0td|jzdzdz|tn||_ |_ ||_||_dS)Nc$t|Sr(r)r*s rr+z+ImageFolder.__init__..is_valid_files*1j999r Tr,rz Found 0 files in subfolders of: rQrR)r<rVr.r/r0r1r5r3r6rXrYrZrErrKrH) r]r<rErrHr+rKr/r=r>r?rs ` rr^zImageFolder.__init__sK  'Jw!!$''  ! : : : : : &bgd&E&E&EFF & &OD!V & &GLLu-- =##&NN1%%% & w<<1  6BF113688J3G3GH )/nnF $ "r rjrklist[_ImageDataType]c|j|}||}|j||}|gS)zn Args: index (int): Index Returns: sample of specific index. rn)r]rjr/ros rrpzImageFolder.__getitem__sA|E"T"" > %^^F++Fxr c*t|jSr(rrrss rrtzImageFolder.__len__ rur rvrw)rjrkrr)rrk)rxryrzr{r|r^rprtrr rrrXsxxt10003333**** 8<:>15/3 %#%#%#%#%#N    !!!!!!r r)rrrrrrr()' __future__rtypingrrrrtyping_extensionsrcollections.abcr paddle._typing.dtype_liker #paddle.vision.transforms.transformsr imager rr|r.PILrr[ paddle.ior paddle.utilsr__all__r%rAr#rkrCrVrrrZr"rrr rrs#""""""888888888888''''''((((((777777>>>>>>&&&&&&$+  %      ###### 1111".X!X!X!X!X!GE"2C"789X!X!X!v """ ===    v!v!v!v!v!'$/01v!v!v!v!v!r