Upload loaders.py with huggingface_hub
Browse files- loaders.py +6 -2
loaders.py
CHANGED
|
@@ -243,10 +243,14 @@ class LoadFromIBMCloud(Loader):
|
|
| 243 |
aws_secret_access_key=self.aws_secret_access_key,
|
| 244 |
endpoint_url=self.endpoint_url,
|
| 245 |
)
|
| 246 |
-
local_dir = os.path.join(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 247 |
if not os.path.exists(local_dir):
|
| 248 |
Path(local_dir).mkdir(parents=True, exist_ok=True)
|
| 249 |
-
|
| 250 |
if isinstance(self.data_files, Mapping):
|
| 251 |
data_files_names = list(self.data_files.values())
|
| 252 |
if not isinstance(data_files_names[0], str):
|
|
|
|
| 243 |
aws_secret_access_key=self.aws_secret_access_key,
|
| 244 |
endpoint_url=self.endpoint_url,
|
| 245 |
)
|
| 246 |
+
local_dir = os.path.join(
|
| 247 |
+
self.cache_dir,
|
| 248 |
+
self.bucket_name,
|
| 249 |
+
self.data_dir,
|
| 250 |
+
f"loader_limit_{self.loader_limit}",
|
| 251 |
+
)
|
| 252 |
if not os.path.exists(local_dir):
|
| 253 |
Path(local_dir).mkdir(parents=True, exist_ok=True)
|
|
|
|
| 254 |
if isinstance(self.data_files, Mapping):
|
| 255 |
data_files_names = list(self.data_files.values())
|
| 256 |
if not isinstance(data_files_names[0], str):
|