Cumulated only for pip

app.py

@@ -6,7 +6,7 @@ from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
 from urllib.parse import parse_qs, urlparse
 
 from huggingface_hub import list_datasets, set_access_token, HfFolder
-from datasets import load_dataset, DatasetDict
+from datasets import load_dataset, DatasetDict, Dataset
 import numpy as np
 
 HF_TOKEN = os.environ['HF_TOKEN']
@@ -20,6 +20,30 @@ datasets = {
     "pip": load_dataset("open-source-metrics/pip").sort('day')
 }
 
+val = 0
+
+
+def _range(e):
+    global val
+    e['range'] = val
+    val += 1
+    return e
+
+
+stars = {}
+for k, v in datasets['stars'].items():
+    stars[k] = v.map(_range)
+    val = 0
+
+issues = {}
+for k, v in datasets['issues'].items():
+    issues[k] = v.map(_range)
+    val = 0
+
+datasets['stars'] = DatasetDict(**stars)
+datasets['issues'] = DatasetDict(**issues)
+
+
 # datasets = {
 #     k1: DatasetDict({
 #         k2: v2.select(range(0, len(v2), max(1, int(len(v2) / 1000)))) for k2, v2 in v1.items()
@@ -27,6 +51,18 @@ datasets = {
 # }
 
 
+def link_values(library_names, returned_values):
+    previous_values = {library_name: None for library_name in library_names}
+    for library_name in library_names:
+        for i in returned_values.keys():
+            if library_name not in returned_values[i]:
+                returned_values[i][library_name] = previous_values[library_name]
+            else:
+                previous_values[library_name] = returned_values[i][library_name]
+
+    return returned_values
+
+
 def running_mean(x, N, total_length=-1):
     cumsum = np.cumsum(np.insert(x, 0, 0))
     to_pad = max(total_length - len(cumsum), 0)
@@ -43,7 +79,6 @@ class RequestHandler(SimpleHTTPRequestHandler):
 
         if self.path.startswith("/initialize"):
             dataset_keys = {k: set(v.keys()) for k, v in datasets.items()}
-            dataset_keys['issues'].remove('transformers')
             dataset_with_most_splits = max([d for d in dataset_keys.values()], key=len)
             warnings = []
 
@@ -68,18 +103,34 @@ class RequestHandler(SimpleHTTPRequestHandler):
             library_names = query.get("input", None)[0]
             library_names = library_names.split(',')
 
-            returned_values = {}
-            for library_name in library_names:
-                for i in datasets['pip'][library_name]:
-                    if i['day'] in returned_values:
-                        returned_values[i['day']][library_name] = i['num_downloads']
-                    else:
-                        returned_values[i['day']] = {library_name: i['num_downloads']}
-
-            for library_name in library_names:
-                for i in returned_values.keys():
-                    if library_name not in returned_values[i]:
-                        returned_values[i][library_name] = None
+            if 'Cumulated' in library_names:
+                dataset_keys = {k: set(v.keys()) for k, v in datasets.items()}
+                dataset_with_most_splits = max([d for d in dataset_keys.values()], key=len)
+                library_names = list(dataset_with_most_splits)
+
+                returned_values = {}
+                for library_name in library_names:
+                    for i in datasets['pip'][library_name]:
+                        if i['day'] in returned_values:
+                            returned_values[i['day']]['Cumulated'] += i['num_downloads']
+                        else:
+                            returned_values[i['day']] = {'Cumulated': i['num_downloads']}
+
+                library_names = ['Cumulated']
+
+            else:
+                returned_values = {}
+                for library_name in library_names:
+                    for i in datasets['pip'][library_name]:
+                        if i['day'] in returned_values:
+                            returned_values[i['day']][library_name] = i['num_downloads']
+                        else:
+                            returned_values[i['day']] = {library_name: i['num_downloads']}
+
+                for library_name in library_names:
+                    for i in returned_values.keys():
+                        if library_name not in returned_values[i]:
+                            returned_values[i][library_name] = None
 
             returned_values = collections.OrderedDict(sorted(returned_values.items()))
             output = {l: [k[l] for k in returned_values.values()] for l in library_names}
@@ -105,23 +156,20 @@ class RequestHandler(SimpleHTTPRequestHandler):
             for library_name in library_names:
                 dataset = dataset_dict[library_name]
 
-                n = 0
                 for i in dataset:
-                    n += 1
                     if i['dates'] in returned_values:
-                        returned_values[i['dates']][library_name] = n
+                        returned_values[i['dates']][library_name] = i['range']
                     else:
-                        returned_values[i['dates']] = {library_name: n}
-
-            for library_name in library_names:
-                for i in returned_values.keys():
-                    if library_name not in returned_values[i]:
-                        returned_values[i][library_name] = None
+                        returned_values[i['dates']] = {library_name: i['range']}
 
             returned_values = collections.OrderedDict(sorted(returned_values.items()))
+            returned_values = link_values(library_names, returned_values)
             output = {l: [k[l] for k in returned_values.values()][::-1] for l in library_names}
             output['day'] = list(returned_values.keys())[::-1]
 
+            # Trim down to a smaller number of points.
+            output = {k: [v for i, v in enumerate(value) if i % int(len(value) / 100) == 0] for k, value in output.items()}
+
             self.send_response(200)
             self.send_header("Content-Type", "application/json")
             self.end_headers()
@@ -142,23 +190,20 @@ class RequestHandler(SimpleHTTPRequestHandler):
             for library_name in library_names:
                 dataset = dataset_dict[library_name]
 
-                n = 0
-                for k, i in enumerate(dataset):
-                    n += 1
+                for i in dataset:
                     if i['dates'] in returned_values:
-                        returned_values[i['dates']][library_name] = n
+                        returned_values[i['dates']][library_name] = i['range']
                     else:
-                        returned_values[i['dates']] = {library_name: n}
-
-            for library_name in library_names:
-                for i in returned_values.keys():
-                    if library_name not in returned_values[i]:
-                        returned_values[i][library_name] = None
+                        returned_values[i['dates']] = {library_name: i['range']}
 
             returned_values = collections.OrderedDict(sorted(returned_values.items()))
+            returned_values = link_values(library_names, returned_values)
             output = {l: [k[l] for k in returned_values.values()][::-1] for l in library_names}
             output['day'] = list(returned_values.keys())[::-1]
 
+            # Trim down to a smaller number of points.
+            output = {k: [v for i, v in enumerate(value) if i % int(len(value) / 100) == 0] for k, value in output.items()}
+
             self.send_response(200)
             self.send_header("Content-Type", "application/json")
             self.end_headers()

index.js

@@ -41,6 +41,7 @@ const createButton = (title, libraries, methods) => {
 const initialize = async () => {
     const inferResponse = await fetch(`initialize`);
     const inferJson = await inferResponse.json();
+    inferJson.push('Cumulated')
     // const graphsDiv = document.getElementsByClassName('graphs')[0];
     const librarySelector = document.getElementById('library-selector');
     const graphSelector = document.getElementById('graph-selector');
@@ -62,7 +63,11 @@ const initialize = async () => {
 
         const checkBoxLabel = document.createElement('label');
         const labelSpan = document.createElement('span')
-        labelSpan.textContent = element.charAt(0).toUpperCase() + element.slice(1)
+
+        if (element == 'Cumulated')
+            labelSpan.textContent = "Cumulated - Only works for pip installs, will crash otherwise."
+        else
+            labelSpan.textContent = element.charAt(0).toUpperCase() + element.slice(1)
         checkBoxLabel.appendChild(checkBox)
         checkBoxLabel.appendChild(labelSpan)