From d7e5997a3587f8b0fa326fb3e22748102568191e Mon Sep 17 00:00:00 2001 From: Yuchen Fan Date: Fri, 18 Aug 2023 16:30:48 -0700 Subject: [PATCH] file to fsspec --- video2dataset/dataloader/custom_wds.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/video2dataset/dataloader/custom_wds.py b/video2dataset/dataloader/custom_wds.py index 6e0e3285..4e825725 100644 --- a/video2dataset/dataloader/custom_wds.py +++ b/video2dataset/dataloader/custom_wds.py @@ -13,7 +13,7 @@ from torch.utils.data import IterableDataset from torch.utils.data.datapipes.iter import IterableWrapper from torch.utils.data.datapipes.utils.common import StreamWrapper -from torchdata.datapipes.iter import S3FileLoader, IterDataPipe, FileOpener +from torchdata.datapipes.iter import S3FileLoader, IterDataPipe, FSSpecFileOpener from torchdata.datapipes.iter import TarArchiveLoader from torchdata.datapipes.utils.common import validate_pathname_binary_tuple @@ -532,7 +532,7 @@ def __init__( main_datapipe = S3FileLoader(main_datapipe, buffer_size=buffer_size) else: # regular fileopener - main_datapipe = FileOpener(main_datapipe, mode="b") + main_datapipe = FSSpecFileOpener(main_datapipe, mode="b") # adapted TarLoader which closes open tarfile handles after exceeding them main_datapipe = TarArchiveLoaderAndCloser(datapipe=main_datapipe, handler=handler).groupby(grouper) if sample_shuffle > 0: