Update app.py

app.py

@@ -1,52 +1,83 @@
+import json
 import os
 import pickle
 import re
-import json
 
 import gradio as gr
 import matplotlib.pyplot as plt
 import networkx as nx
 from tqdm import tqdm
 
+
 def load_json_from_path(path):
     with open(path, "r", encoding="utf8") as f:
         obj = json.loads(f.read())
 
     return obj
 
+
 class Visualizer:
 
     def __init__(self, cache_root="."):
+        self.iso_codes_to_names = load_json_from_path(os.path.join(cache_root, "iso_to_fullname.json"))
+        for code in self.iso_codes_to_names:
+            self.iso_codes_to_names[code] = re.sub("\(.*?\)", "", self.iso_codes_to_names[code])
+
         tree_lookup_path = os.path.join(cache_root, "lang_1_to_lang_2_to_tree_dist.json")
-        self.tree_dist = load_json_from_path(tree_lookup_path)
+        tree_dist = load_json_from_path(tree_lookup_path)
+        distances = list()
+        for lang_1 in tree_dist:
+            if lang_1 not in self.iso_codes_to_names:
+                continue
+            for lang_2 in tree_dist[lang_1]:
+                if lang_2 not in self.iso_codes_to_names:
+                    continue
+                if lang_1 != lang_2:
+                    distances.append((self.iso_codes_to_names[lang_1], self.iso_codes_to_names[lang_2], tree_dist[lang_1][lang_2]))
+        min_dist = min(d for _, _, d in distances)
+        max_dist = max(d for _, _, d in distances)
+        self.tree_distances = [(entity1, entity2, (d - min_dist) / (max_dist - min_dist)) for entity1, entity2, d in distances]
 
         map_lookup_path = os.path.join(cache_root, "lang_1_to_lang_2_to_map_dist.json")
-        self.map_dist = load_json_from_path(map_lookup_path)
-        largest_value_map_dist = 0.0
-        for _, values in self.map_dist.items():
-            for _, value in values.items():
-                largest_value_map_dist = max(largest_value_map_dist, value)
-        for key1 in self.map_dist:
-            for key2 in self.map_dist[key1]:
-                self.map_dist[key1][key2] = self.map_dist[key1][key2] / largest_value_map_dist
+        map_dist = load_json_from_path(map_lookup_path)
+        distances = list()
+        for lang_1 in map_dist:
+            if lang_1 not in self.iso_codes_to_names:
+                continue
+            for lang_2 in map_dist[lang_1]:
+                if lang_2 not in self.iso_codes_to_names:
+                    continue
+                if lang_1 != lang_2:
+                    distances.append((self.iso_codes_to_names[lang_1], self.iso_codes_to_names[lang_2], map_dist[lang_1][lang_2]))
+        min_dist = min(d for _, _, d in distances)
+        max_dist = max(d for _, _, d in distances)
+        self.map_distances = [(entity1, entity2, (d - min_dist) / (max_dist - min_dist)) for entity1, entity2, d in distances]
 
         asp_dict_path = os.path.join(cache_root, "asp_dict.pkl")
         with open(asp_dict_path, 'rb') as dictfile:
             asp_sim = pickle.load(dictfile)
         lang_list = list(asp_sim.keys())
-        self.asp_dist = dict()
+        asp_dist = dict()
         seen_langs = set()
         for lang_1 in lang_list:
             if lang_1 not in seen_langs:
                 seen_langs.add(lang_1)
-                self.asp_dist[lang_1] = dict()
+                asp_dist[lang_1] = dict()
             for index, lang_2 in enumerate(lang_list):
                 if lang_2 not in seen_langs:  # it's symmetric
-                    self.asp_dist[lang_1][lang_2] = 1 - asp_sim[lang_1][index]
-
-        self.iso_codes_to_names = load_json_from_path(os.path.join(cache_root, "iso_to_fullname.json"))
-        for code in self.iso_codes_to_names:
-            self.iso_codes_to_names[code] = re.sub("\(.*?\)", "", self.iso_codes_to_names[code])
+                    asp_dist[lang_1][lang_2] = 1 - asp_sim[lang_1][index]
+        distances = list()
+        for lang_1 in asp_dist:
+            if lang_1 not in self.iso_codes_to_names:
+                continue
+            for lang_2 in asp_dist[lang_1]:
+                if lang_2 not in self.iso_codes_to_names:
+                    continue
+                if lang_1 != lang_2:
+                    distances.append((self.iso_codes_to_names[lang_1], self.iso_codes_to_names[lang_2], asp_dist[lang_1][lang_2]))
+        min_dist = min(d for _, _, d in distances)
+        max_dist = max(d for _, _, d in distances)
+        self.asp_distances = [(entity1, entity2, (d - min_dist) / (max_dist - min_dist)) for entity1, entity2, d in distances]
 
     def visualize(self, distance_type, neighbor, num_neighbors):
         plt.clf()
@@ -56,46 +87,30 @@ class Visualizer:
                                  "Distance to the Lowest Common Ancestor in the Language Family Tree",
                                  "Angular Distance between the Frequencies of Phonemes"]
         if distance_type == "Distance to the Lowest Common Ancestor in the Language Family Tree":
-            distance_measure = self.tree_dist
+            normalized_distances = self.tree_distances
         elif distance_type == "Angular Distance between the Frequencies of Phonemes":
-            distance_measure = self.asp_dist
+            normalized_distances = self.asp_distances
         elif distance_type == "Physical Distance between Language Centroids on the Globe":
-            distance_measure = self.map_dist
-
-        distances = list()
-
-        for lang_1 in distance_measure:
-            if lang_1 not in self.iso_codes_to_names:
-                continue
-            for lang_2 in distance_measure[lang_1]:
-                if lang_2 not in self.iso_codes_to_names:
-                    continue
-                distances.append((self.iso_codes_to_names[lang_1], self.iso_codes_to_names[lang_2], distance_measure[lang_1][lang_2]))
+            normalized_distances = self.map_distances
 
         G = nx.Graph()
-        min_dist = min(d for _, _, d in distances)
-        max_dist = max(d for _, _, d in distances)
-        normalized_distances = [(entity1, entity2, (d - min_dist) / (max_dist - min_dist)) for entity1, entity2, d in distances]
-
         d_dist = list()
         for entity1, entity2, d in tqdm(normalized_distances):
             if neighbor == entity2 or neighbor == entity1:
-                if entity1 != entity2:
-                    d_dist.append(d)
+                d_dist.append(d)
         thresh = sorted(d_dist)[num_neighbors]
         neighbors = set()
         for entity1, entity2, d in tqdm(normalized_distances):
-            if d < thresh and (neighbor == entity2 or neighbor == entity1) and (entity1 != entity2):
+            if d <= thresh and (neighbor == entity2 or neighbor == entity1) and len(neighbors) < num_neighbors + 1:
                 neighbors.add(entity1)
                 neighbors.add(entity2)
-                spring_tension = (thresh - d) * 10  # for vis purposes
+                spring_tension = (thresh - d) * 100  # for vis purposes
                 G.add_edge(entity1, entity2, weight=spring_tension)
         neighbors.remove(neighbor)
         for entity1, entity2, d in tqdm(normalized_distances):
             if entity2 in neighbors and entity1 in neighbors:
-                if entity1 != entity2:
-                    spring_tension = thresh - d
-                    G.add_edge(entity1, entity2, weight=spring_tension)
+                spring_tension = thresh - d
+                G.add_edge(entity1, entity2, weight=spring_tension)
 
         pos = nx.spring_layout(G, weight="weight")  # Positions for all nodes
         edges = G.edges(data=True)
@@ -106,7 +121,7 @@ class Visualizer:
         # nx.draw_networkx_edges(G, pos, edgelist=edges_not_connected_to_specific_node, edge_color='gray', alpha=0.1, width=1)
         for u, v, d in edges:
             if u == neighbor or v == neighbor:
-                nx.draw_networkx_edge_labels(G, pos, edge_labels={(u, v): round((thresh - (d['weight'] / 10)) * 10, 2)}, font_color="red", alpha=0.4)  # reverse modifications
+                nx.draw_networkx_edge_labels(G, pos, edge_labels={(u, v): round((thresh - (d['weight'] / 100)) * 10, 2)}, font_color="red", alpha=0.4)  # reverse modifications
         nx.draw_networkx_labels(G, pos, font_size=14, font_family='sans-serif', font_color='green')
         nx.draw_networkx_labels(G, pos, labels={neighbor: neighbor}, font_size=14, font_family='sans-serif', font_color='red')
         plt.title(f'Graph of {distance_type}')
@@ -116,7 +131,7 @@ class Visualizer:
 
 
 if __name__ == '__main__':
-    vis = Visualizer(cache_root=".")
+    vis = Visualizer(cache_root="Preprocessing/multilinguality")
     text_selection = [f"{vis.iso_codes_to_names[iso_code]}" for iso_code in vis.iso_codes_to_names]
     iface = gr.Interface(fn=vis.visualize,
                          inputs=[gr.Dropdown(["Physical Distance between Language Centroids on the Globe",