Upload dataset.py with huggingface_hub
Browse files- dataset.py +6 -4
dataset.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
| 1 |
-
import logging
|
| 2 |
import os
|
| 3 |
|
| 4 |
import datasets
|
|
@@ -20,6 +19,8 @@ from .hf_utils import __file__ as _
|
|
| 20 |
from .instructions import __file__ as _
|
| 21 |
from .load import __file__ as _
|
| 22 |
from .loaders import __file__ as _
|
|
|
|
|
|
|
| 23 |
from .metric import __file__ as _
|
| 24 |
from .metrics import __file__ as _
|
| 25 |
from .normalizers import __file__ as _
|
|
@@ -30,7 +31,6 @@ from .random_utils import __file__ as _
|
|
| 30 |
from .recipe import __file__ as _
|
| 31 |
from .register import __file__ as _
|
| 32 |
from .register import _reset_env_local_catalogs, register_all_artifacts
|
| 33 |
-
from .renderers import __file__ as _
|
| 34 |
from .schema import __file__ as _
|
| 35 |
from .split_utils import __file__ as _
|
| 36 |
from .splitters import __file__ as _
|
|
@@ -45,6 +45,8 @@ from .validate import __file__ as _
|
|
| 45 |
from .version import __file__ as _
|
| 46 |
from .version import version
|
| 47 |
|
|
|
|
|
|
|
| 48 |
__default_recipe__ = "standard_recipe"
|
| 49 |
|
| 50 |
|
|
@@ -114,10 +116,10 @@ class Dataset(datasets.GeneratorBasedBuilder):
|
|
| 114 |
unitxt_installed = False
|
| 115 |
|
| 116 |
if unitxt_installed:
|
| 117 |
-
|
| 118 |
dataset = get_dataset_artifact_installed(self.config.name)
|
| 119 |
else:
|
| 120 |
-
|
| 121 |
dataset = get_dataset_artifact(self.config.name)
|
| 122 |
|
| 123 |
self._generators = dataset()
|
|
|
|
|
|
|
| 1 |
import os
|
| 2 |
|
| 3 |
import datasets
|
|
|
|
| 19 |
from .instructions import __file__ as _
|
| 20 |
from .load import __file__ as _
|
| 21 |
from .loaders import __file__ as _
|
| 22 |
+
from .logging_utils import __file__ as _
|
| 23 |
+
from .logging_utils import get_logger
|
| 24 |
from .metric import __file__ as _
|
| 25 |
from .metrics import __file__ as _
|
| 26 |
from .normalizers import __file__ as _
|
|
|
|
| 31 |
from .recipe import __file__ as _
|
| 32 |
from .register import __file__ as _
|
| 33 |
from .register import _reset_env_local_catalogs, register_all_artifacts
|
|
|
|
| 34 |
from .schema import __file__ as _
|
| 35 |
from .split_utils import __file__ as _
|
| 36 |
from .splitters import __file__ as _
|
|
|
|
| 45 |
from .version import __file__ as _
|
| 46 |
from .version import version
|
| 47 |
|
| 48 |
+
logger = get_logger()
|
| 49 |
+
|
| 50 |
__default_recipe__ = "standard_recipe"
|
| 51 |
|
| 52 |
|
|
|
|
| 116 |
unitxt_installed = False
|
| 117 |
|
| 118 |
if unitxt_installed:
|
| 119 |
+
logger.info("Loading with installed unitxt library...")
|
| 120 |
dataset = get_dataset_artifact_installed(self.config.name)
|
| 121 |
else:
|
| 122 |
+
logger.info("Loading with huggingface unitxt copy...")
|
| 123 |
dataset = get_dataset_artifact(self.config.name)
|
| 124 |
|
| 125 |
self._generators = dataset()
|