Akos Hadnagy commited on
Commit
fc3c4b8
·
1 Parent(s): 77dbb67

Make sure cache is skipped for HF datasets

Browse files
Files changed (1) hide show
  1. data.py +2 -2
data.py CHANGED
@@ -60,12 +60,12 @@ def read_one_dataframe(json_path: str, device_label: str) -> pd.DataFrame:
60
  def get_distant_data() -> pd.DataFrame:
61
  # Retrieve AMD dataframe
62
  amd_src = "hf://datasets/optimum-amd/transformers_daily_ci/**/runs/**/ci_results_run_models_gpu/model_results.json"
63
- files_amd = sorted(fs.glob(amd_src), reverse=True)
64
  df_amd = read_one_dataframe(f"hf://{files_amd[0]}", "amd")
65
  # Retrieve NVIDIA dataframe, which pattern should be:
66
  # hf://datasets/hf-internal-testing/transformers_daily_ci/raw/main/YYYY-MM-DD/ci_results_run_models_gpu/model_results.json
67
  nvidia_src = "hf://datasets/hf-internal-testing/transformers_daily_ci/*/ci_results_run_models_gpu/model_results.json"
68
- files_nvidia = sorted(fs.glob(nvidia_src), reverse=True)
69
  # NOTE: should this be removeprefix instead of lstrip?
70
  nvidia_path = files_nvidia[0].lstrip('datasets/hf-internal-testing/transformers_daily_ci/')
71
  nvidia_path = "https://huggingface.co/datasets/hf-internal-testing/transformers_daily_ci/raw/main/" + nvidia_path
 
60
  def get_distant_data() -> pd.DataFrame:
61
  # Retrieve AMD dataframe
62
  amd_src = "hf://datasets/optimum-amd/transformers_daily_ci/**/runs/**/ci_results_run_models_gpu/model_results.json"
63
+ files_amd = sorted(fs.glob(amd_src, refresh=True), reverse=True)
64
  df_amd = read_one_dataframe(f"hf://{files_amd[0]}", "amd")
65
  # Retrieve NVIDIA dataframe, which pattern should be:
66
  # hf://datasets/hf-internal-testing/transformers_daily_ci/raw/main/YYYY-MM-DD/ci_results_run_models_gpu/model_results.json
67
  nvidia_src = "hf://datasets/hf-internal-testing/transformers_daily_ci/*/ci_results_run_models_gpu/model_results.json"
68
+ files_nvidia = sorted(fs.glob(nvidia_src, refresh=True), reverse=True)
69
  # NOTE: should this be removeprefix instead of lstrip?
70
  nvidia_path = files_nvidia[0].lstrip('datasets/hf-internal-testing/transformers_daily_ci/')
71
  nvidia_path = "https://huggingface.co/datasets/hf-internal-testing/transformers_daily_ci/raw/main/" + nvidia_path