Static space work

app.py

@@ -12,11 +12,13 @@ HF_TOKEN = os.environ['HF_TOKEN']
 set_access_token(HF_TOKEN)
 HfFolder.save_token(HF_TOKEN)
 
-datasets_to_load = [
-    "open-source-metrics/stars",
-    "open-source-metrics/issues",
-    
-]
+
+datasets = {
+    # "stars": load_dataset("open-source-metrics/stars"),
+    "issues": load_dataset("open-source-metrics/issues"),
+    "pip": load_dataset("open-source-metrics/pip")
+}
+
 
 def running_mean(x, N, total_length=-1):
     cumsum = np.cumsum(np.insert(x, 0, 0))
@@ -33,14 +35,20 @@ class RequestHandler(SimpleHTTPRequestHandler):
             return SimpleHTTPRequestHandler.do_GET(self)
 
         if self.path.startswith("/initialize"):
-            metrics_datasets = [d.id.split('/')[1] for d in list_datasets(author='open-source-metrics', use_auth_token=True)]
-            pip_installs_datasets = [d.split('-')[0] for d in metrics_datasets if 'pip' in d]
+            dataset_keys = {k: set(v.keys()) for k, v in datasets.items()}
+            dataset_keys['issues'].remove('transformers')
+            dataset_with_most_splits = max([d for d in dataset_keys.values()], key=len)
+            warnings = []
+
+            for k, v in dataset_keys.items():
+                if len(v.keys()) < len(dataset_with_most_splits):
+                    warnings.extend(f"The {k} dataset does not contain all splits. Missing: {dataset_with_most_splits - v}")
 
             self.send_response(200)
             self.send_header("Content-Type", "application/json")
             self.end_headers()
 
-            self.wfile.write(json.dumps(pip_installs_datasets).encode("utf-8"))
+            self.wfile.write(json.dumps(dataset_keys).encode("utf-8"))
 
             return SimpleHTTPRequestHandler