Akos Hadnagy
commited on
Commit
·
fc3c4b8
1
Parent(s):
77dbb67
Make sure cache is skipped for HF datasets
Browse files
data.py
CHANGED
|
@@ -60,12 +60,12 @@ def read_one_dataframe(json_path: str, device_label: str) -> pd.DataFrame:
|
|
| 60 |
def get_distant_data() -> pd.DataFrame:
|
| 61 |
# Retrieve AMD dataframe
|
| 62 |
amd_src = "hf://datasets/optimum-amd/transformers_daily_ci/**/runs/**/ci_results_run_models_gpu/model_results.json"
|
| 63 |
-
files_amd = sorted(fs.glob(amd_src), reverse=True)
|
| 64 |
df_amd = read_one_dataframe(f"hf://{files_amd[0]}", "amd")
|
| 65 |
# Retrieve NVIDIA dataframe, which pattern should be:
|
| 66 |
# hf://datasets/hf-internal-testing/transformers_daily_ci/raw/main/YYYY-MM-DD/ci_results_run_models_gpu/model_results.json
|
| 67 |
nvidia_src = "hf://datasets/hf-internal-testing/transformers_daily_ci/*/ci_results_run_models_gpu/model_results.json"
|
| 68 |
-
files_nvidia = sorted(fs.glob(nvidia_src), reverse=True)
|
| 69 |
# NOTE: should this be removeprefix instead of lstrip?
|
| 70 |
nvidia_path = files_nvidia[0].lstrip('datasets/hf-internal-testing/transformers_daily_ci/')
|
| 71 |
nvidia_path = "https://huggingface.co/datasets/hf-internal-testing/transformers_daily_ci/raw/main/" + nvidia_path
|
|
|
|
| 60 |
def get_distant_data() -> pd.DataFrame:
|
| 61 |
# Retrieve AMD dataframe
|
| 62 |
amd_src = "hf://datasets/optimum-amd/transformers_daily_ci/**/runs/**/ci_results_run_models_gpu/model_results.json"
|
| 63 |
+
files_amd = sorted(fs.glob(amd_src, refresh=True), reverse=True)
|
| 64 |
df_amd = read_one_dataframe(f"hf://{files_amd[0]}", "amd")
|
| 65 |
# Retrieve NVIDIA dataframe, which pattern should be:
|
| 66 |
# hf://datasets/hf-internal-testing/transformers_daily_ci/raw/main/YYYY-MM-DD/ci_results_run_models_gpu/model_results.json
|
| 67 |
nvidia_src = "hf://datasets/hf-internal-testing/transformers_daily_ci/*/ci_results_run_models_gpu/model_results.json"
|
| 68 |
+
files_nvidia = sorted(fs.glob(nvidia_src, refresh=True), reverse=True)
|
| 69 |
# NOTE: should this be removeprefix instead of lstrip?
|
| 70 |
nvidia_path = files_nvidia[0].lstrip('datasets/hf-internal-testing/transformers_daily_ci/')
|
| 71 |
nvidia_path = "https://huggingface.co/datasets/hf-internal-testing/transformers_daily_ci/raw/main/" + nvidia_path
|