Upload folder using huggingface_hub

app.py

@@ -395,11 +395,11 @@ with gr.Blocks(title="Model Test Results Dashboard", css=load_css()) as demo:
         try:
             Ci_results.load_historical_data(start_date, end_date)
             if Ci_results.historical_df.empty:
-                return gr.update(), "No historical data found for the selected date range."
+                return gr.update(), f"No historical data found for the selected date range ({start_date} to {end_date}). Please try a different date range."
             
             # Create time-series summary plot
             time_series_plot = create_time_series_summary(Ci_results.historical_df)
-            return time_series_plot, f"Loaded historical data from {start_date} to {end_date}"
+            return time_series_plot, f"Loaded historical data from {start_date} to {end_date} ({len(Ci_results.historical_df)} records)"
         except Exception as e:
             logger.error(f"Error loading historical data: {e}")
             return gr.update(), f"Error loading historical data: {str(e)}"
@@ -414,7 +414,7 @@ with gr.Blocks(title="Model Test Results Dashboard", css=load_css()) as demo:
     def show_time_series_model(selected_model):
         """Show time-series view for a specific model."""
         if Ci_results.historical_df.empty:
-            return gr.update(), "No historical data loaded. Please load historical data first."
+            return gr.update()
         
         try:
             time_series_plot = create_model_time_series(Ci_results.historical_df, selected_model)

data.py

@@ -102,57 +102,57 @@ def read_one_dataframe(json_path: str, device_label: str) -> tuple[pd.DataFrame,
 def get_available_dates() -> List[str]:
     """Get list of available dates from both AMD and NVIDIA datasets."""
     try:
-        # Get AMD dates
-        amd_src = "hf://datasets/optimum-amd/transformers_daily_ci/**/runs/**/ci_results_run_models_gpu/model_results.json"
+        # Get AMD dates - the path structure is: YYYY-MM-DD/ci_results_run_models_gpu/model_results.json
+        amd_src = "hf://datasets/optimum-amd/transformers_daily_ci/*/ci_results_run_models_gpu/model_results.json"
         files_amd = sorted(fs.glob(amd_src, refresh=True), reverse=True)
+        logger.info(f"Found {len(files_amd)} AMD files")
         
-        # Get NVIDIA dates
+        # Get NVIDIA dates - same structure
         nvidia_src = "hf://datasets/hf-internal-testing/transformers_daily_ci/*/ci_results_run_models_gpu/model_results.json"
         files_nvidia = sorted(fs.glob(nvidia_src, refresh=True), reverse=True)
+        logger.info(f"Found {len(files_nvidia)} NVIDIA files")
         
         # Extract dates from file paths
         amd_dates = set()
         for file_path in files_amd:
-            pattern = r'transformers_daily_ci(.*?)/(\d{4}-\d{2}-\d{2})'
+            # Pattern to match the date in the path: YYYY-MM-DD/ci_results_run_models_gpu/model_results.json
+            pattern = r'transformers_daily_ci/(\d{4}-\d{2}-\d{2})/ci_results_run_models_gpu/model_results\.json'
             match = re.search(pattern, file_path)
             if match:
-                amd_dates.add(match.group(2))
+                amd_dates.add(match.group(1))
         
         nvidia_dates = set()
         for file_path in files_nvidia:
-            pattern = r'transformers_daily_ci/(\d{4}-\d{2}-\d{2})'
+            # Same pattern for NVIDIA
+            pattern = r'transformers_daily_ci/(\d{4}-\d{2}-\d{2})/ci_results_run_models_gpu/model_results\.json'
             match = re.search(pattern, file_path)
             if match:
                 nvidia_dates.add(match.group(1))
         
+        logger.info(f"AMD dates: {sorted(amd_dates, reverse=True)[:5]}...")  # Show first 5
+        logger.info(f"NVIDIA dates: {sorted(nvidia_dates, reverse=True)[:5]}...")  # Show first 5
+        
         # Return intersection of both datasets (dates where both have data)
         common_dates = sorted(amd_dates.intersection(nvidia_dates), reverse=True)
+        logger.info(f"Common dates: {len(common_dates)} dates where both AMD and NVIDIA have data")
         return common_dates[:30]  # Limit to last 30 days for performance
         
     except Exception as e:
         logger.error(f"Error getting available dates: {e}")
-        # Return sample dates for fallback
-        today = datetime.now()
-        return [(today - timedelta(days=i)).strftime("%Y-%m-%d") for i in range(7)]
+        # Return empty list if no data available
+        return []
 
 
 def get_data_for_date(target_date: str) -> tuple[pd.DataFrame, str]:
     """Get data for a specific date."""
     try:
-        # Construct paths for specific date
-        amd_src = f"hf://datasets/optimum-amd/transformers_daily_ci/**/runs/{target_date}/**/ci_results_run_models_gpu/model_results.json"
+        # Construct paths for specific date - correct structure: YYYY-MM-DD/ci_results_run_models_gpu/model_results.json
+        amd_src = f"hf://datasets/optimum-amd/transformers_daily_ci/{target_date}/ci_results_run_models_gpu/model_results.json"
         nvidia_src = f"hf://datasets/hf-internal-testing/transformers_daily_ci/{target_date}/ci_results_run_models_gpu/model_results.json"
         
-        # Find matching files
-        files_amd = fs.glob(amd_src, refresh=True)
-        files_nvidia = fs.glob(nvidia_src, refresh=True)
-        
-        if not files_amd or not files_nvidia:
-            raise FileNotFoundError(f"No data found for date {target_date}")
-        
-        # Use the first matching file for each
-        df_amd, _ = read_one_dataframe(f"hf://{files_amd[0]}", "amd")
-        df_nvidia, _ = read_one_dataframe(f"https://huggingface.co/datasets/hf-internal-testing/transformers_daily_ci/raw/main/{target_date}/ci_results_run_models_gpu/model_results.json", "nvidia")
+        # Read dataframes directly
+        df_amd, _ = read_one_dataframe(amd_src, "amd")
+        df_nvidia, _ = read_one_dataframe(nvidia_src, "nvidia")
         
         # Join both dataframes
         joined = df_amd.join(df_nvidia, rsuffix="_nvidia", lsuffix="_amd", how="outer")
@@ -167,8 +167,8 @@ def get_data_for_date(target_date: str) -> tuple[pd.DataFrame, str]:
         
     except Exception as e:
         logger.error(f"Error getting data for date {target_date}: {e}")
-        # Fallback to sample data
-        return get_sample_data()
+        # Return empty dataframe instead of sample data for historical functionality
+        return pd.DataFrame(), target_date
 
 
 def get_historical_data(start_date: str, end_date: str) -> pd.DataFrame:
@@ -184,16 +184,21 @@ def get_historical_data(start_date: str, end_date: str) -> pd.DataFrame:
             date_str = current_dt.strftime("%Y-%m-%d")
             try:
                 df, _ = get_data_for_date(date_str)
-                df['date'] = date_str
-                historical_data.append(df)
-                logger.info(f"Loaded data for {date_str}")
+                # Only add non-empty dataframes
+                if not df.empty:
+                    df['date'] = date_str
+                    historical_data.append(df)
+                    logger.info(f"Loaded data for {date_str}")
+                else:
+                    logger.warning(f"No data available for {date_str}")
             except Exception as e:
                 logger.warning(f"Could not load data for {date_str}: {e}")
             
             current_dt += timedelta(days=1)
         
         if not historical_data:
-            raise ValueError("No historical data found for the specified range")
+            logger.warning("No historical data found for the specified range")
+            return pd.DataFrame()
         
         # Combine all dataframes
         combined_df = pd.concat(historical_data, ignore_index=False)
@@ -317,6 +322,10 @@ class CIResults:
         try:
             self.available_dates = get_available_dates()
             logger.info(f"Available dates: {len(self.available_dates)} dates")
+            if self.available_dates:
+                logger.info(f"Date range: {self.available_dates[-1]} to {self.available_dates[0]}")
+            else:
+                logger.warning("No available dates found")
         except Exception as e:
             logger.error(f"Error loading available dates: {e}")
             self.available_dates = []