Initial Commit

app.py

@@ -0,0 +1,26 @@
+import pandas as pd
+import numpy as np
+
+import gradio as gr
+import pygwalker as pyg
+
+
+from pygwalker.api.gradio import PYGWALKER_ROUTE, get_html_on_gradio
+
+
+from data import df
+from network import analysis
+
+
+with gr.Blocks() as demo:
+  # with gr.Tab("📝 About"):
+  #     # gr.Markdown(ABOUT_TEXT)
+  with gr.Tab("📊 Dashboard"):
+    gr.Label("Visually explore witches family data")
+    gr.Markdown("You can use drag-and-drop operations to explore the data, start your analysis now!")
+    pyg_app = get_html_on_gradio(df, spec="./config.json")
+    gr.HTML(pyg_app)
+  with gr.Tab("👪 Family Connection"):
+    gr.HTML(analysis)
+            
+demo.launch(app_kwargs={"routes": [PYGWALKER_ROUTE]}).queue()

config.json

@@ -0,0 +1,251 @@
+[
+    {
+      "config": {
+        "defaultAggregated": true,
+        "geoms": [
+          "poi"
+        ],
+        "coordSystem": "geographic",
+        "limit": -1
+      },
+      "encodings": {
+        "dimensions": [
+          {
+            "dragId": "gw_0oFN",
+            "fid": "GW_153M2HTVSG",
+            "name": "item",
+            "basename": "item",
+            "semanticType": "nominal",
+            "analyticType": "dimension"
+          },
+          {
+            "dragId": "gw_TfYx",
+            "fid": "GW_2X73E2RH6IPJOOK4G8G",
+            "name": "accusedurl",
+            "basename": "accusedurl",
+            "semanticType": "nominal",
+            "analyticType": "dimension"
+          },
+          {
+            "dragId": "gw_MK3G",
+            "fid": "GW_G0O5DCTUJ01U6ZK8G",
+            "name": "itemLabel",
+            "basename": "itemLabel",
+            "semanticType": "nominal",
+            "analyticType": "dimension"
+          },
+          {
+            "dragId": "gw_4Ud1",
+            "fid": "GW_6RT7I4HK7073D9RPOESM5CH80",
+            "name": "residence",
+            "basename": "residence",
+            "semanticType": "nominal",
+            "analyticType": "dimension"
+          },
+          {
+            "dragId": "gw_fW-1",
+            "fid": "GW_M27TBC9KBOWJRWX5WS34",
+            "name": "gender",
+            "basename": "gender",
+            "semanticType": "nominal",
+            "analyticType": "dimension"
+          },
+          {
+            "dragId": "gw_YRLn",
+            "fid": "GW_2ZE8EA85LWGIU48N1GW",
+            "name": "class",
+            "basename": "class",
+            "semanticType": "nominal",
+            "analyticType": "dimension"
+          },
+          {
+            "dragId": "gw_WKTM",
+            "fid": "GW_5498MX0A38QWHP82WQA4T7971KAQCP16WRJZZU8",
+            "name": "place_of_detention",
+            "basename": "place_of_detention",
+            "semanticType": "nominal",
+            "analyticType": "dimension"
+          },
+          {
+            "dragId": "gw_u9Xa",
+            "fid": "GW_R8BM6NZ296ISPYXOCTXCNQEAVA6142MCW7RHG7RLLBK26Z9POPYCZ4",
+            "name": "manner_of_inhumane_treatment",
+            "basename": "manner_of_inhumane_treatment",
+            "semanticType": "nominal",
+            "analyticType": "dimension"
+          },
+          {
+            "dragId": "gw_DUxe",
+            "fid": "GW_2AJUGNLE6MEPTZNHZAYG8IEPPSCE4MO1C",
+            "name": "cause_of_death",
+            "basename": "cause_of_death",
+            "semanticType": "nominal",
+            "analyticType": "dimension"
+          },
+          {
+            "dragId": "gw_wtxd",
+            "fid": "GW_1AW6HAPIZKT3VHC18C33DQM3968",
+            "name": "occupation",
+            "basename": "occupation",
+            "semanticType": "nominal",
+            "analyticType": "dimension"
+          },
+          {
+            "dragId": "gw_EEfP",
+            "fid": "GW_OMQ0RHNFV2PIFPFR6A0G",
+            "name": "spouse",
+            "basename": "spouse",
+            "semanticType": "nominal",
+            "analyticType": "dimension"
+          },
+          {
+            "dragId": "gw_AtER",
+            "fid": "GW_GGYX5I6DWC5400P4G",
+            "name": "longitude",
+            "basename": "longitude",
+            "semanticType": "quantitative",
+            "analyticType": "dimension"
+          },
+          {
+            "dragId": "gw_q-LU",
+            "fid": "GW_2BCEDJ2KYSIN2ZHS",
+            "name": "latitude",
+            "basename": "latitude",
+            "semanticType": "quantitative",
+            "analyticType": "dimension"
+          },
+          {
+            "fid": "GW_1K1RWLJMRKVG0",
+            "name": "father",
+            "semanticType": "nominal",
+            "analyticType": "dimension",
+            "basename": "father",
+            "dragId": "GW_BFrlUJKd"
+          },
+          {
+            "fid": "GW_CHAWSEHZCMSO4W",
+            "name": "sibling",
+            "semanticType": "nominal",
+            "analyticType": "dimension",
+            "basename": "sibling",
+            "dragId": "GW_enUhIyMF"
+          },
+          {
+            "fid": "GW_7NIDHER5RU8",
+            "name": "child",
+            "semanticType": "nominal",
+            "analyticType": "dimension",
+            "basename": "child",
+            "dragId": "GW_c7Noxzdq"
+          },
+          {
+            "fid": "GW_1NWT9FT2YJVAO",
+            "name": "mother",
+            "semanticType": "nominal",
+            "analyticType": "dimension",
+            "basename": "mother",
+            "dragId": "GW_YApxzLXH"
+          }
+        ],
+        "measures": [
+          {
+            "dragId": "gw_count_fid",
+            "fid": "gw_count_fid",
+            "name": "Row count",
+            "analyticType": "measure",
+            "semanticType": "quantitative",
+            "aggName": "sum",
+            "computed": true,
+            "expression": {
+              "op": "one",
+              "params": [],
+              "as": "gw_count_fid"
+            }
+          },
+          {
+            "dragId": "gw_mea_val_fid",
+            "fid": "GW_4KYB9KVI3CK37NQAUJ65UJWVK",
+            "name": "Measure values",
+            "analyticType": "measure",
+            "semanticType": "quantitative",
+            "aggName": "sum"
+          }
+        ],
+        "rows": [],
+        "columns": [],
+        "color": [
+          {
+            "dragId": "gw_I8g7",
+            "fid": "GW_2ZE8EA85LWGIU48N1GW",
+            "name": "class",
+            "basename": "class",
+            "semanticType": "nominal",
+            "analyticType": "dimension"
+          }
+        ],
+        "opacity": [],
+        "size": [],
+        "shape": [],
+        "radius": [],
+        "theta": [],
+        "longitude": [
+          {
+            "dragId": "gw_y2hj",
+            "fid": "GW_GGYX5I6DWC5400P4G",
+            "name": "longitude",
+            "basename": "longitude",
+            "semanticType": "quantitative",
+            "analyticType": "dimension"
+          }
+        ],
+        "latitude": [
+          {
+            "dragId": "gw_vshP",
+            "fid": "GW_2BCEDJ2KYSIN2ZHS",
+            "name": "latitude",
+            "basename": "latitude",
+            "semanticType": "quantitative",
+            "analyticType": "dimension"
+          }
+        ],
+        "geoId": [],
+        "details": [
+          {
+            "dragId": "gw_KQnh",
+            "fid": "GW_6RT7I4HK7073D9RPOESM5CH80",
+            "name": "residence",
+            "basename": "residence",
+            "semanticType": "nominal",
+            "analyticType": "dimension"
+          }
+        ],
+        "filters": [],
+        "text": []
+      },
+      "layout": {
+        "showActions": false,
+        "showTableSummary": false,
+        "stack": "stack",
+        "interactiveScale": false,
+        "zeroScale": true,
+        "size": {
+          "mode": "auto",
+          "width": 800,
+          "height": 800
+        },
+        "format": {},
+        "geoKey": "name",
+        "resolve": {
+          "x": false,
+          "y": false,
+          "color": false,
+          "opacity": false,
+          "shape": false,
+          "size": false
+        },
+        "scaleIncludeUnmatchedChoropleth": false
+      },
+      "visId": "gw_b4Sl",
+      "name": "Chart 1"
+    }
+  ]

data.py

@@ -0,0 +1,68 @@
+import sys
+import pandas as pd
+
+from SPARQLWrapper import SPARQLWrapper, JSON
+
+
+endpoint_url = "https://query.wikidata.org/sparql"
+
+query = """
+# Places of residence of accused witches in Scotland 1563-1736
+SELECT ?accusedurl ?item ?itemLabel ?residenceLabel ?genderLabel ?occupationLabel ?classLabel ?manner_of_inhumane_treatmentLabel ?place_of_detentionLabel ?cause_of_deathLabel ?fatherLabel ?motherLabel ?siblingLabel ?childLabel ?spouseLabel ?coords WHERE {
+  ?item wdt:P31 wd:Q5;
+    wdt:P4478 ?accused.
+  wd:P4478 wdt:P1630 ?formatterurl.
+  BIND(IRI(REPLACE(?accused, "^(.+)$", ?formatterurl)) AS ?accusedurl)
+  ?item wdt:P551 ?residence.
+  ?residence wdt:P625 ?coords.
+  
+  OPTIONAL { ?item wdt:P21 ?gender. }
+  OPTIONAL { ?item wdt:P106 ?occupation. }
+  OPTIONAL { ?item wdt:P3716 ?class. }
+  SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". }
+  OPTIONAL { ?item wdt:P7160 ?manner_of_inhumane_treatment. }
+  OPTIONAL { ?item wdt:P2632 ?place_of_detention. }
+  OPTIONAL { ?item wdt:P509 ?cause_of_death. }
+  OPTIONAL { ?item wdt:P22 ?father. }
+  OPTIONAL { ?item wdt:P25 ?mother. }
+  OPTIONAL { ?item wdt:P3373 ?sibling. }
+  OPTIONAL { ?item wdt:P40 ?child. }
+  OPTIONAL { ?item wdt:P26 ?spouse. }
+  OPTIONAL { ?item wdt:P551 ?residence. }
+}
+"""
+
+
+def get_results(endpoint_url, query):
+  """
+  Obtain SPARQL query results.
+  """
+  user_agent = "WDQS-example Python/%s.%s" % (sys.version_info[0], sys.version_info[1])
+  # TODO adjust user agent; see https://w.wiki/CX6
+  sparql = SPARQLWrapper(endpoint_url, agent=user_agent)
+  sparql.setQuery(query)
+  sparql.setReturnFormat(JSON)
+  return sparql.query().convert()
+
+
+def load_data():
+  """
+  Obtain data for accused witches charged with witchcraft.
+  """
+  lst = []
+  results = get_results(endpoint_url, query)
+  for result in results["results"]["bindings"]:
+      d = {}
+      for k, v in result.items():
+          d[k] = v['value']
+      lst.append(d)
+  data = pd.DataFrame(lst)
+  data.dropna(subset=['siblingLabel', 'spouseLabel', 'childLabel', 'fatherLabel', 'motherLabel'], how="all", inplace=True)
+  data['longitude'] = data['coords'].str.replace("Point", "").apply(lambda x: x.split()[0].lstrip("(")).astype(float)
+  data['latitude'] = data['coords'].str.replace("Point", "").apply(lambda x: x.split()[-1].rstrip(")")).astype(float)
+  data.drop(['coords'], axis=1, inplace=True)
+  data.columns = [col.replace("Label", "") if col != "itemLabel" else col for col in data.columns.tolist()]
+  return data
+
+
+df = load_data()

network.py

@@ -0,0 +1,61 @@
+import pandas as pd
+import networkx as nx
+import matplotlib.pyplot as plt
+from pyvis.network import Network
+
+from data import df
+
+
+def analysis():
+    G = nx.DiGraph()
+
+    # Define the columns to consider for relationships
+    relationship_columns = ['father', 'sibling', 'spouse', 'mother', 'child']
+
+    G.add_nodes_from(df["itemLabel"])
+
+    for index, row in df.iterrows():
+        main_entity = row['itemLabel']
+        for relationship in relationship_columns:
+            if pd.notna(row[relationship]):
+                G.add_edge(main_entity, row[relationship], relationship=str(relationship), label=relationship)
+
+    plt.figure(figsize=(50, 20))  # Set the size of the plot
+    # pos = nx.kamada_kawai_layout(G)  # Compute the positions of the nodes
+
+    # # Draw the nodes and edges with labels
+    # nx.draw(G, pos, with_labels=True, node_size=20, node_color='skyblue', font_size=10, font_color='black', font_weight='bold', alpha=0.7)
+
+    # # Draw edge labels (the relationships)
+    # edge_labels = nx.get_edge_attributes(G, 'relationship')
+    # nx.draw_networkx_edge_labels(G, pos, edge_labels=edge_labels, font_color='red')
+
+    # plt.title('Relationship Graph with Labels')
+    # plt.axis('off')  # Hide the axes for clarity
+    # plt.show()
+
+
+    net = Network(bgcolor="#222222", font_color="white", directed=True)
+    net.from_nx(G)
+
+    for node in net.nodes:
+        node["title"] = node["id"]
+        node["value"] = len(G[node["id"]])
+
+
+    for edge in net.edges:
+        # Set edge title to the relationship from your graph
+        relationship = G[edge["from"]][edge["to"]].get("relationship", "Unknown")
+        edge["title"] = relationship  # This will show as a tooltip
+        edge["color"] = "blue"
+        edge["width"] = 2 if relationship != "Unknown" else 1
+
+    html = net.generate_html()
+    #need to remove ' from HTML
+    html = html.replace("'", "\"")
+    
+    return f"""<iframe style="width: 100%; height: 1000px;margin:0 auto" name="result" allow="midi; geolocation; microphone; camera; 
+    display-capture; encrypted-media;" sandbox="allow-modals allow-forms 
+    allow-scripts allow-same-origin allow-popups 
+    allow-top-navigation-by-user-activation allow-downloads" allowfullscreen="" 
+    allowpaymentrequest="" frameborder="0" srcdoc='{html}'></iframe>"""

requirements.txt

@@ -0,0 +1,7 @@
+pandas
+sparqlwrapper
+gradio
+pygwalker
+datasets
+pyvis
+networkx