Add issues

app.py

@@ -6,7 +6,7 @@ from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
 from urllib.parse import parse_qs, urlparse
 
 from huggingface_hub import list_datasets, set_access_token, HfFolder
-from datasets import load_dataset
+from datasets import load_dataset, DatasetDict
 import numpy as np
 
 HF_TOKEN = os.environ['HF_TOKEN']
@@ -15,11 +15,17 @@ HfFolder.save_token(HF_TOKEN)
 
 
 datasets = {
-    # "stars": load_dataset("open-source-metrics/stars"),
-    "issues": load_dataset("open-source-metrics/issues"),
+    "stars": load_dataset("open-source-metrics/stars").sort('dates'),
+    "issues": load_dataset("open-source-metrics/issues").sort('dates'),
     "pip": load_dataset("open-source-metrics/pip").sort('day')
 }
 
+# datasets = {
+#     k1: DatasetDict({
+#         k2: v2.select(range(0, len(v2), max(1, int(len(v2) / 1000)))) for k2, v2 in v1.items()
+#     }) for k1, v1 in datasets.items()
+# }
+
 
 def running_mean(x, N, total_length=-1):
     cumsum = np.cumsum(np.insert(x, 0, 0))
@@ -94,17 +100,50 @@ class RequestHandler(SimpleHTTPRequestHandler):
             library_names = library_names.split(',')
 
             returned_values = {}
-            dataset_dict = load_dataset(f"open-source-metrics/stars", use_auth_token=True).sort('dates')
+            dataset_dict = datasets['stars']
 
             for library_name in library_names:
                 dataset = dataset_dict[library_name]
 
                 n = 0
-                for k, i in enumerate(dataset):
-                    # Decimate values if there are too many
-                    if len(dataset) > 1000 and k % int(len(dataset) / 1000) != 0:
-                        continue
+                for i in dataset:
+                    n += 1
+                    if i['dates'] in returned_values:
+                        returned_values[i['dates']][library_name] = n
+                    else:
+                        returned_values[i['dates']] = {library_name: n}
+
+            for library_name in library_names:
+                for i in returned_values.keys():
+                    if library_name not in returned_values[i]:
+                        returned_values[i][library_name] = None
+
+            returned_values = collections.OrderedDict(sorted(returned_values.items()))
+            output = {l: [k[l] for k in returned_values.values()][::-1] for l in library_names}
+            output['day'] = list(returned_values.keys())[::-1]
+
+            self.send_response(200)
+            self.send_header("Content-Type", "application/json")
+            self.end_headers()
+
+            self.wfile.write(json.dumps(output).encode("utf-8"))
+
+            return SimpleHTTPRequestHandler
+
+        if self.path.startswith("/retrieveIssues"):
+            url = urlparse(self.path)
+            query = parse_qs(url.query)
+            library_names = query.get("input", None)[0]
+            library_names = library_names.split(',')
 
+            returned_values = {}
+            dataset_dict = datasets['issues']
+
+            for library_name in library_names:
+                dataset = dataset_dict[library_name]
+
+                n = 0
+                for k, i in enumerate(dataset):
                     n += 1
                     if i['dates'] in returned_values:
                         returned_values[i['dates']][library_name] = n
@@ -131,8 +170,8 @@ class RequestHandler(SimpleHTTPRequestHandler):
         return SimpleHTTPRequestHandler.do_GET(self)
 
 
-server = ThreadingHTTPServer(("", 7860), RequestHandler)
+server = ThreadingHTTPServer(("", 7861), RequestHandler)
 
-print("Running on port 7860")
+print("Running on port 7861")
 
 server.serve_forever()

index.html

@@ -18,5 +18,6 @@
         <div id="selector-submit" class="submit"></div>
         <canvas id="pip-graph"></canvas>
         <canvas id="star-graph"></canvas>
+        <canvas id="issue-graph"></canvas>
     </body>
 </html>

index.js

@@ -70,7 +70,7 @@ const initialize = async () => {
         librarySelector.appendChild(div)
     }
 
-    for (const element of ['pip', 'stars']) {
+    for (const element of ['pip', 'stars', 'issues']) {
         const div = document.createElement('div');
         const checkBox = document.createElement('input');
         checkBox.type = 'checkbox'
@@ -87,7 +87,7 @@ const initialize = async () => {
     }
 
     const fetchButton = createButton('Fetch', inferJson, () => {
-        const graphNames = ['pip', 'stars'].filter(e => document.querySelector(`#${e}CheckboxGraph`).checked);
+        const graphNames = ['pip', 'stars', 'issues'].filter(e => document.querySelector(`#${e}CheckboxGraph`).checked);
         const graphs = []
 
         if (graphNames.includes('pip'))
@@ -96,6 +96,9 @@ const initialize = async () => {
         if (graphNames.includes('stars'))
             graphs.push(retrieveStars)
 
+        if (graphNames.includes('issues'))
+            graphs.push(retrieveIssues)
+
         return graphs
     })
     selectorSubmit.appendChild(fetchButton);
@@ -140,6 +143,12 @@ const retrievePipInstalls = async (libraryNames) => {
                     type: 'time',
                 }
             },
+            plugins: {
+                title: {
+                    display: true,
+                    text: 'Pip installs'
+                }
+            }
         }
     });
     return myChart;
@@ -172,6 +181,58 @@ const retrieveStars = async (libraryNames) => {
     const ctx = document.getElementById('star-graph');
 
     const myChart = new Chart(ctx, {
+        title: "Stars",
+        type: 'line',
+        data: {labels, datasets},
+        options: {
+            scales: {
+                y: {
+                    beginAtZero: true
+                },
+                x: {
+                    type: 'time',
+                }
+            },
+            plugins: {
+                title: {
+                    display: true,
+                    text: 'Number of stargazers'
+                }
+            }
+        }
+    });
+    return myChart;
+};
+
+const retrieveIssues = async (libraryNames) => {
+    const inferResponse = await fetch(`retrieveIssues?input=${libraryNames}`);
+    const inferJson = await inferResponse.json();
+    const colors = ['Lilac', 'Red', 'Blue', 'Orange', 'Green']
+
+    console.log(inferJson)
+    const labels = Array.from(inferJson['day']).map(e => new Date(e))
+    const datasets = [];
+    for (const element in inferJson) {
+        if (element === 'day')
+            continue
+
+        const color = colors.pop()
+        datasets.push({
+            label: element,
+            data: inferJson[element],
+            backgroundColor: color,
+            borderColor: color,
+            tension: 0.01,
+            pointRadius: 1,
+            borderWidth: 2,
+            fill: false
+        })
+    }
+
+    const ctx = document.getElementById('issue-graph');
+
+    const myChart = new Chart(ctx, {
+        title: "Issues",
         type: 'line',
         data: {labels, datasets},
         options: {
@@ -183,6 +244,12 @@ const retrieveStars = async (libraryNames) => {
                     type: 'time',
                 }
             },
+            plugins: {
+                title: {
+                    display: true,
+                    text: 'Number of issues, PRs, and comments on these'
+                }
+            }
         }
     });
     return myChart;