Adding divergence graph

app.py

@@ -14,9 +14,11 @@ from tabs.trader_plots import (
     plot_trader_metrics_by_trader_type,
     default_trader_metric,
     trader_metric_choices,
-    get_trader_metrics_text,
+    get_metrics_text,
 )
 
+from tabs.market_plots import plot_kl_div_per_market
+
 
 def get_logger():
     logger = logging.getLogger(__name__)
@@ -37,7 +39,7 @@ logger = get_logger()
 
 def get_all_data():
     """
-    Get parquet file from weekly stats
+    Get parquet files from weekly stats and new generated
     """
     logger.info("Getting traders data")
     con = duckdb.connect(":memory:")
@@ -49,14 +51,22 @@ def get_all_data():
     df1 = con.execute(query1).fetchdf()
     logger.info("Got all data from all_trades_profitability.parquet")
 
+    # Query to fetch data from closed_markets_div.parquet
+    query2 = f"""
+    SELECT *
+    FROM read_parquet('./data/closed_markets_div.parquet')
+    """
+    df2 = con.execute(query2).fetchdf()
+    logger.info("Got all data from closed_markets_div.parquet")
+
     con.close()
 
-    return df1
+    return df1, df2
 
 
 def prepare_data():
 
-    all_trades = get_all_data()
+    all_trades, closed_markets = get_all_data()
 
     all_trades["creation_date"] = all_trades["creation_timestamp"].dt.date
 
@@ -81,10 +91,14 @@ def prepare_data():
     trader_agents_data["month_year_week"] = (
         trader_agents_data["creation_timestamp"].dt.to_period("W").dt.strftime("%b-%d")
     )
-    return trader_agents_data
+
+    closed_markets["month_year_week"] = (
+        closed_markets["opening_datetime"].dt.to_period("W").dt.strftime("%b-%d")
+    )
+    return trader_agents_data, closed_markets
 
 
-trader_agents_data = prepare_data()
+trader_agents_data, closed_markets = prepare_data()
 print("trader agents data before computing metrics")
 print(trader_agents_data.head())
 demo = gr.Blocks()
@@ -122,7 +136,7 @@ with demo:
                         traders_df=weekly_metrics_by_market_creator,
                     )
                 with gr.Column(scale=1):
-                    trade_details_text = get_trader_metrics_text()
+                    trade_details_text = get_metrics_text()
 
             def update_trader_details(trader_detail):
                 return plot_trader_metrics_by_market_creator(
@@ -154,7 +168,7 @@ with demo:
                         traders_df=weekly_metrics_by_trader_type,
                     )
                 with gr.Column(scale=1):
-                    trader_metrics_text = get_trader_metrics_text()
+                    trader_metrics_text = get_metrics_text()
 
             def update_trader_metric(trader_metric):
                 return plot_trader_metrics_by_trader_type(
@@ -167,5 +181,19 @@ with demo:
                 inputs=trader_metric_selector,
                 outputs=trader_type_plot,
             )
+        with gr.TabItem("📉Closed Markets Kullback–Leibler divergence"):
+            with gr.Row():
+                gr.Markdown(
+                    "# Weekly Kullback–Leibler divergence computed for the closed markets"
+                )
+            with gr.Row():
+                gr.Markdown(
+                    "This divergence is a type of statistical distance between two probability distributions P and Q. In our case P is the probability defined by the final liquidity distribution of the market. While Q is the distribution of the final outcome."
+                )
+            with gr.Row():
+                with gr.Column(scale=3):
+                    kl_div_plot = plot_kl_div_per_market(closed_markets=closed_markets)
+                with gr.Column(scale=1):
+                    metrics_text = get_metrics_text()
 
 demo.queue(default_concurrency_limit=40).launch()

data/closed_markets_div.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01028e48165f8e468cd377da59e13da584a0938cdc64549dee2a1c523d6e1b13
+size 48695

data/fpmms.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86135bb64013c54d5180c31fca13235943eb39571e760a695dac2aaa1e9cb1ce
+size 436427

notebooks/closed_markets.ipynb

@@ -0,0 +1,1481 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "try:\n",
+    "    markets = pd.read_parquet(\"../data/fpmms.parquet\")\n",
+    "except Exception:\n",
+    "    print(\"Error reading the parquet file\")\n",
+    "\n",
+    "markets[\"currentAnswer\"] = markets[\"currentAnswer\"].apply(lambda x: x.lower())\n",
+    "# filter only markets with  yes, no answers\n",
+    "valid_answers = [\"yes\", \"no\"]\n",
+    "markets = markets.loc[markets[\"currentAnswer\"].isin(valid_answers)]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "4686"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(markets)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "4686"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(markets.id.unique())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>currentAnswer</th>\n",
+       "      <th>id</th>\n",
+       "      <th>title</th>\n",
+       "      <th>market_creator</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>no</td>\n",
+       "      <td>0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f5</td>\n",
+       "      <td>Will the first floating offshore wind research...</td>\n",
+       "      <td>quickstart</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>no</td>\n",
+       "      <td>0x0020d13c89140b47e10db54cbd53852b90bc1391</td>\n",
+       "      <td>Will the Francis Scott Key Bridge in Baltimore...</td>\n",
+       "      <td>quickstart</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>no</td>\n",
+       "      <td>0x003ae5e007cc38b3f86b0ed7c82f938a1285ac07</td>\n",
+       "      <td>Will FC Saarbrucken reach the final of the Ger...</td>\n",
+       "      <td>quickstart</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>yes</td>\n",
+       "      <td>0x004c8d4c619dc6b9caa940f5ea7ef699ae85359c</td>\n",
+       "      <td>Will the pro-life activists convicted for 'con...</td>\n",
+       "      <td>quickstart</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>yes</td>\n",
+       "      <td>0x005e3f7a90585acbec807425a750fbba1d0c2b5c</td>\n",
+       "      <td>Will Apple announce the release of a new M4 ch...</td>\n",
+       "      <td>quickstart</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "  currentAnswer                                          id  \\\n",
+       "0            no  0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f5   \n",
+       "1            no  0x0020d13c89140b47e10db54cbd53852b90bc1391   \n",
+       "2            no  0x003ae5e007cc38b3f86b0ed7c82f938a1285ac07   \n",
+       "3           yes  0x004c8d4c619dc6b9caa940f5ea7ef699ae85359c   \n",
+       "4           yes  0x005e3f7a90585acbec807425a750fbba1d0c2b5c   \n",
+       "\n",
+       "                                               title market_creator  \n",
+       "0  Will the first floating offshore wind research...     quickstart  \n",
+       "1  Will the Francis Scott Key Bridge in Baltimore...     quickstart  \n",
+       "2  Will FC Saarbrucken reach the final of the Ger...     quickstart  \n",
+       "3  Will the pro-life activists convicted for 'con...     quickstart  \n",
+       "4  Will Apple announce the release of a new M4 ch...     quickstart  "
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "markets.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "trades = pd.read_parquet(\"../data/fpmmTrades.parquet\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>collateralAmount</th>\n",
+       "      <th>collateralAmountUSD</th>\n",
+       "      <th>collateralToken</th>\n",
+       "      <th>creationTimestamp</th>\n",
+       "      <th>trader_address</th>\n",
+       "      <th>feeAmount</th>\n",
+       "      <th>id</th>\n",
+       "      <th>oldOutcomeTokenMarginalPrice</th>\n",
+       "      <th>outcomeIndex</th>\n",
+       "      <th>outcomeTokenMarginalPrice</th>\n",
+       "      <th>...</th>\n",
+       "      <th>market_creator</th>\n",
+       "      <th>fpmm.answerFinalizedTimestamp</th>\n",
+       "      <th>fpmm.arbitrationOccurred</th>\n",
+       "      <th>fpmm.currentAnswer</th>\n",
+       "      <th>fpmm.id</th>\n",
+       "      <th>fpmm.isPendingArbitration</th>\n",
+       "      <th>fpmm.openingTimestamp</th>\n",
+       "      <th>fpmm.outcomes</th>\n",
+       "      <th>fpmm.title</th>\n",
+       "      <th>fpmm.condition.id</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>450426474650738688</td>\n",
+       "      <td>0.4504269694034145716308073094168006</td>\n",
+       "      <td>0xe91d153e0b41518a2ce8dd3d7944fa863463a97d</td>\n",
+       "      <td>1724553455</td>\n",
+       "      <td>0x022b36c50b85b8ae7addfb8a35d76c59d5814834</td>\n",
+       "      <td>9008529493014773</td>\n",
+       "      <td>0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f50x02...</td>\n",
+       "      <td>0.592785210609610270634125335572129</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0.6171295391012242250994586583534301</td>\n",
+       "      <td>...</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>1725071760</td>\n",
+       "      <td>False</td>\n",
+       "      <td>0x00000000000000000000000000000000000000000000...</td>\n",
+       "      <td>0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f5</td>\n",
+       "      <td>False</td>\n",
+       "      <td>1724976000</td>\n",
+       "      <td>[Yes, No]</td>\n",
+       "      <td>Will the first floating offshore wind research...</td>\n",
+       "      <td>0x0e940f12f30e928e4879c52d065d9da739a3d3f020d1...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>610163214546941400</td>\n",
+       "      <td>0.6101636232215150135654007337015298</td>\n",
+       "      <td>0xe91d153e0b41518a2ce8dd3d7944fa863463a97d</td>\n",
+       "      <td>1724811940</td>\n",
+       "      <td>0x034c4ad84f7ac6638bf19300d5bbe7d9b981e736</td>\n",
+       "      <td>12203264290938828</td>\n",
+       "      <td>0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f50x03...</td>\n",
+       "      <td>0.842992636523755061934822129394812</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0.8523396372892128845826889719620915</td>\n",
+       "      <td>...</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>1725071760</td>\n",
+       "      <td>False</td>\n",
+       "      <td>0x00000000000000000000000000000000000000000000...</td>\n",
+       "      <td>0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f5</td>\n",
+       "      <td>False</td>\n",
+       "      <td>1724976000</td>\n",
+       "      <td>[Yes, No]</td>\n",
+       "      <td>Will the first floating offshore wind research...</td>\n",
+       "      <td>0x0e940f12f30e928e4879c52d065d9da739a3d3f020d1...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>789065092332460672</td>\n",
+       "      <td>0.7890644120527324071908793822796086</td>\n",
+       "      <td>0xe91d153e0b41518a2ce8dd3d7944fa863463a97d</td>\n",
+       "      <td>1724815755</td>\n",
+       "      <td>0x09e9d42a029e8b0c2df3871709a762117a681d92</td>\n",
+       "      <td>15781301846649213</td>\n",
+       "      <td>0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f50x09...</td>\n",
+       "      <td>0.7983775743712442891104598770339028</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0.8152123711444691659642000374025623</td>\n",
+       "      <td>...</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>1725071760</td>\n",
+       "      <td>False</td>\n",
+       "      <td>0x00000000000000000000000000000000000000000000...</td>\n",
+       "      <td>0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f5</td>\n",
+       "      <td>False</td>\n",
+       "      <td>1724976000</td>\n",
+       "      <td>[Yes, No]</td>\n",
+       "      <td>Will the first floating offshore wind research...</td>\n",
+       "      <td>0x0e940f12f30e928e4879c52d065d9da739a3d3f020d1...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>1000000000000000000</td>\n",
+       "      <td>1.000000605383660329048491794939126</td>\n",
+       "      <td>0xe91d153e0b41518a2ce8dd3d7944fa863463a97d</td>\n",
+       "      <td>1724546620</td>\n",
+       "      <td>0x09e9d42a029e8b0c2df3871709a762117a681d92</td>\n",
+       "      <td>20000000000000000</td>\n",
+       "      <td>0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f50x09...</td>\n",
+       "      <td>0.5110745907733438805447072252622708</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0.5746805204222762335911904727318937</td>\n",
+       "      <td>...</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>1725071760</td>\n",
+       "      <td>False</td>\n",
+       "      <td>0x00000000000000000000000000000000000000000000...</td>\n",
+       "      <td>0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f5</td>\n",
+       "      <td>False</td>\n",
+       "      <td>1724976000</td>\n",
+       "      <td>[Yes, No]</td>\n",
+       "      <td>Will the first floating offshore wind research...</td>\n",
+       "      <td>0x0e940f12f30e928e4879c52d065d9da739a3d3f020d1...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>100000000000000000</td>\n",
+       "      <td>0.1000004271262862419547394646567906</td>\n",
+       "      <td>0xe91d153e0b41518a2ce8dd3d7944fa863463a97d</td>\n",
+       "      <td>1724771260</td>\n",
+       "      <td>0x0d049dcaece0ecb6fc81a460da7bcc2a4785d6e5</td>\n",
+       "      <td>2000000000000000</td>\n",
+       "      <td>0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f50x0d...</td>\n",
+       "      <td>0.2713968218662319388988681987389408</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.2804586217805511523845593360379658</td>\n",
+       "      <td>...</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>1725071760</td>\n",
+       "      <td>False</td>\n",
+       "      <td>0x00000000000000000000000000000000000000000000...</td>\n",
+       "      <td>0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f5</td>\n",
+       "      <td>False</td>\n",
+       "      <td>1724976000</td>\n",
+       "      <td>[Yes, No]</td>\n",
+       "      <td>Will the first floating offshore wind research...</td>\n",
+       "      <td>0x0e940f12f30e928e4879c52d065d9da739a3d3f020d1...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>5 rows × 24 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "      collateralAmount                   collateralAmountUSD  \\\n",
+       "0   450426474650738688  0.4504269694034145716308073094168006   \n",
+       "1   610163214546941400  0.6101636232215150135654007337015298   \n",
+       "2   789065092332460672  0.7890644120527324071908793822796086   \n",
+       "3  1000000000000000000   1.000000605383660329048491794939126   \n",
+       "4   100000000000000000  0.1000004271262862419547394646567906   \n",
+       "\n",
+       "                              collateralToken creationTimestamp  \\\n",
+       "0  0xe91d153e0b41518a2ce8dd3d7944fa863463a97d        1724553455   \n",
+       "1  0xe91d153e0b41518a2ce8dd3d7944fa863463a97d        1724811940   \n",
+       "2  0xe91d153e0b41518a2ce8dd3d7944fa863463a97d        1724815755   \n",
+       "3  0xe91d153e0b41518a2ce8dd3d7944fa863463a97d        1724546620   \n",
+       "4  0xe91d153e0b41518a2ce8dd3d7944fa863463a97d        1724771260   \n",
+       "\n",
+       "                               trader_address          feeAmount  \\\n",
+       "0  0x022b36c50b85b8ae7addfb8a35d76c59d5814834   9008529493014773   \n",
+       "1  0x034c4ad84f7ac6638bf19300d5bbe7d9b981e736  12203264290938828   \n",
+       "2  0x09e9d42a029e8b0c2df3871709a762117a681d92  15781301846649213   \n",
+       "3  0x09e9d42a029e8b0c2df3871709a762117a681d92  20000000000000000   \n",
+       "4  0x0d049dcaece0ecb6fc81a460da7bcc2a4785d6e5   2000000000000000   \n",
+       "\n",
+       "                                                  id  \\\n",
+       "0  0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f50x02...   \n",
+       "1  0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f50x03...   \n",
+       "2  0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f50x09...   \n",
+       "3  0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f50x09...   \n",
+       "4  0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f50x0d...   \n",
+       "\n",
+       "           oldOutcomeTokenMarginalPrice outcomeIndex  \\\n",
+       "0   0.592785210609610270634125335572129            1   \n",
+       "1   0.842992636523755061934822129394812            1   \n",
+       "2  0.7983775743712442891104598770339028            1   \n",
+       "3  0.5110745907733438805447072252622708            1   \n",
+       "4  0.2713968218662319388988681987389408            0   \n",
+       "\n",
+       "              outcomeTokenMarginalPrice  ... market_creator  \\\n",
+       "0  0.6171295391012242250994586583534301  ...     quickstart   \n",
+       "1  0.8523396372892128845826889719620915  ...     quickstart   \n",
+       "2  0.8152123711444691659642000374025623  ...     quickstart   \n",
+       "3  0.5746805204222762335911904727318937  ...     quickstart   \n",
+       "4  0.2804586217805511523845593360379658  ...     quickstart   \n",
+       "\n",
+       "  fpmm.answerFinalizedTimestamp fpmm.arbitrationOccurred  \\\n",
+       "0                    1725071760                    False   \n",
+       "1                    1725071760                    False   \n",
+       "2                    1725071760                    False   \n",
+       "3                    1725071760                    False   \n",
+       "4                    1725071760                    False   \n",
+       "\n",
+       "                                  fpmm.currentAnswer  \\\n",
+       "0  0x00000000000000000000000000000000000000000000...   \n",
+       "1  0x00000000000000000000000000000000000000000000...   \n",
+       "2  0x00000000000000000000000000000000000000000000...   \n",
+       "3  0x00000000000000000000000000000000000000000000...   \n",
+       "4  0x00000000000000000000000000000000000000000000...   \n",
+       "\n",
+       "                                      fpmm.id fpmm.isPendingArbitration  \\\n",
+       "0  0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f5                     False   \n",
+       "1  0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f5                     False   \n",
+       "2  0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f5                     False   \n",
+       "3  0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f5                     False   \n",
+       "4  0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f5                     False   \n",
+       "\n",
+       "   fpmm.openingTimestamp fpmm.outcomes  \\\n",
+       "0             1724976000     [Yes, No]   \n",
+       "1             1724976000     [Yes, No]   \n",
+       "2             1724976000     [Yes, No]   \n",
+       "3             1724976000     [Yes, No]   \n",
+       "4             1724976000     [Yes, No]   \n",
+       "\n",
+       "                                          fpmm.title  \\\n",
+       "0  Will the first floating offshore wind research...   \n",
+       "1  Will the first floating offshore wind research...   \n",
+       "2  Will the first floating offshore wind research...   \n",
+       "3  Will the first floating offshore wind research...   \n",
+       "4  Will the first floating offshore wind research...   \n",
+       "\n",
+       "                                   fpmm.condition.id  \n",
+       "0  0x0e940f12f30e928e4879c52d065d9da739a3d3f020d1...  \n",
+       "1  0x0e940f12f30e928e4879c52d065d9da739a3d3f020d1...  \n",
+       "2  0x0e940f12f30e928e4879c52d065d9da739a3d3f020d1...  \n",
+       "3  0x0e940f12f30e928e4879c52d065d9da739a3d3f020d1...  \n",
+       "4  0x0e940f12f30e928e4879c52d065d9da739a3d3f020d1...  \n",
+       "\n",
+       "[5 rows x 24 columns]"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "trades.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<class 'pandas.core.frame.DataFrame'>\n",
+      "RangeIndex: 26835 entries, 0 to 26834\n",
+      "Data columns (total 24 columns):\n",
+      " #   Column                         Non-Null Count  Dtype \n",
+      "---  ------                         --------------  ----- \n",
+      " 0   collateralAmount               26835 non-null  object\n",
+      " 1   collateralAmountUSD            26835 non-null  object\n",
+      " 2   collateralToken                26835 non-null  object\n",
+      " 3   creationTimestamp              26835 non-null  object\n",
+      " 4   trader_address                 26835 non-null  object\n",
+      " 5   feeAmount                      26835 non-null  object\n",
+      " 6   id                             26835 non-null  object\n",
+      " 7   oldOutcomeTokenMarginalPrice   26835 non-null  object\n",
+      " 8   outcomeIndex                   26835 non-null  object\n",
+      " 9   outcomeTokenMarginalPrice      26835 non-null  object\n",
+      " 10  outcomeTokensTraded            26835 non-null  object\n",
+      " 11  title                          26835 non-null  object\n",
+      " 12  transactionHash                26835 non-null  object\n",
+      " 13  type                           26835 non-null  object\n",
+      " 14  market_creator                 26835 non-null  object\n",
+      " 15  fpmm.answerFinalizedTimestamp  24829 non-null  object\n",
+      " 16  fpmm.arbitrationOccurred       26835 non-null  bool  \n",
+      " 17  fpmm.currentAnswer             24829 non-null  object\n",
+      " 18  fpmm.id                        26835 non-null  object\n",
+      " 19  fpmm.isPendingArbitration      26835 non-null  bool  \n",
+      " 20  fpmm.openingTimestamp          26835 non-null  object\n",
+      " 21  fpmm.outcomes                  26835 non-null  object\n",
+      " 22  fpmm.title                     26835 non-null  object\n",
+      " 23  fpmm.condition.id              26835 non-null  object\n",
+      "dtypes: bool(2), object(22)\n",
+      "memory usage: 4.6+ MB\n"
+     ]
+    }
+   ],
+   "source": [
+    "trades.info()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Index(['collateralAmount', 'collateralAmountUSD', 'collateralToken',\n",
+       "       'creationTimestamp', 'trader_address', 'feeAmount', 'id',\n",
+       "       'oldOutcomeTokenMarginalPrice', 'outcomeIndex',\n",
+       "       'outcomeTokenMarginalPrice', 'outcomeTokensTraded', 'title',\n",
+       "       'transactionHash', 'type', 'market_creator',\n",
+       "       'fpmm.answerFinalizedTimestamp', 'fpmm.arbitrationOccurred',\n",
+       "       'fpmm.currentAnswer', 'fpmm.id', 'fpmm.isPendingArbitration',\n",
+       "       'fpmm.openingTimestamp', 'fpmm.outcomes', 'fpmm.title',\n",
+       "       'fpmm.condition.id'],\n",
+       "      dtype='object')"
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "trades.columns"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "markets = list(trades[\"fpmm.id\"].unique())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "803"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(markets)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 50,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/var/folders/gp/02mb1d514ng739czlxw1lhh00000gn/T/ipykernel_3094/2495807215.py:12: SettingWithCopyWarning: \n",
+      "A value is trying to be set on a copy of a slice from a DataFrame\n",
+      "\n",
+      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
+      "  trade_markets.rename(\n"
+     ]
+    }
+   ],
+   "source": [
+    "from datetime import datetime\n",
+    "INVALID_ANSWER_HEX = (\n",
+    "    \"0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff\"\n",
+    ")\n",
+    "columns_of_interest = [\n",
+    "        \"fpmm.currentAnswer\",\n",
+    "        \"fpmm.id\",\n",
+    "        \"fpmm.openingTimestamp\",\n",
+    "        \"market_creator\",\n",
+    "    ]\n",
+    "trade_markets = trades[columns_of_interest]\n",
+    "trade_markets.rename(\n",
+    "    columns={\n",
+    "        \"fpmm.currentAnswer\": \"currentAnswer\",\n",
+    "        \"fpmm.openingTimestamp\": \"openingTimestamp\",\n",
+    "        \"fpmm.id\": \"id\",\n",
+    "    },\n",
+    "    inplace=True,\n",
+    ")\n",
+    "trade_markets = trade_markets.drop_duplicates(subset=['id'], keep='last')\n",
+    "# remove invalid answers\n",
+    "\n",
+    "trade_markets = trade_markets.loc[trade_markets[\"currentAnswer\"]!= INVALID_ANSWER_HEX]\n",
+    "trade_markets[\"currentAnswer\"] = trade_markets[\"currentAnswer\"].apply(\n",
+    "    lambda x: convert_hex_to_int(x)\n",
+    ")\n",
+    "trade_markets[\"opening_datetime\"] = trade_markets[\"openingTimestamp\"].apply(\n",
+    "    lambda x: datetime.fromtimestamp(int(x))\n",
+    ")\n",
+    "trade_markets = trade_markets.sort_values(by=\"opening_datetime\", ascending=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 63,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "648"
+      ]
+     },
+     "execution_count": 63,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(trade_markets.id.unique())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 64,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "648"
+      ]
+     },
+     "execution_count": 64,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(trade_markets)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 52,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>currentAnswer</th>\n",
+       "      <th>id</th>\n",
+       "      <th>openingTimestamp</th>\n",
+       "      <th>market_creator</th>\n",
+       "      <th>opening_datetime</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>15736</th>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0x92ed80e541f642b564f992245abe640282dd273c</td>\n",
+       "      <td>1727568000</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>2024-09-29 02:00:00</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6272</th>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0x4002481fe7bc39c1baa4b5988c038da13ed05832</td>\n",
+       "      <td>1727568000</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>2024-09-29 02:00:00</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>24383</th>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0xf820d06509027c309b00cd386055982d9bea0c10</td>\n",
+       "      <td>1727568000</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>2024-09-29 02:00:00</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>12418</th>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0x74e0fa941341ebe980fbdcfa8b40244cb448eb56</td>\n",
+       "      <td>1727568000</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>2024-09-29 02:00:00</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4754</th>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0x2f44e179b5cc964e504046bac31d6945a0652af2</td>\n",
+       "      <td>1727568000</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>2024-09-29 02:00:00</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "       currentAnswer                                          id  \\\n",
+       "15736            NaN  0x92ed80e541f642b564f992245abe640282dd273c   \n",
+       "6272             NaN  0x4002481fe7bc39c1baa4b5988c038da13ed05832   \n",
+       "24383            NaN  0xf820d06509027c309b00cd386055982d9bea0c10   \n",
+       "12418            NaN  0x74e0fa941341ebe980fbdcfa8b40244cb448eb56   \n",
+       "4754             NaN  0x2f44e179b5cc964e504046bac31d6945a0652af2   \n",
+       "\n",
+       "      openingTimestamp market_creator    opening_datetime  \n",
+       "15736       1727568000     quickstart 2024-09-29 02:00:00  \n",
+       "6272        1727568000     quickstart 2024-09-29 02:00:00  \n",
+       "24383       1727568000     quickstart 2024-09-29 02:00:00  \n",
+       "12418       1727568000     quickstart 2024-09-29 02:00:00  \n",
+       "4754        1727568000     quickstart 2024-09-29 02:00:00  "
+      ]
+     },
+     "execution_count": 52,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "trade_markets.tail()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 53,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "719"
+      ]
+     },
+     "execution_count": 53,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(trade_markets)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 54,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "trade_markets.dropna(inplace=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 55,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "648"
+      ]
+     },
+     "execution_count": 55,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(trade_markets)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 56,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>currentAnswer</th>\n",
+       "      <th>id</th>\n",
+       "      <th>openingTimestamp</th>\n",
+       "      <th>market_creator</th>\n",
+       "      <th>opening_datetime</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>20792</th>\n",
+       "      <td>1.0</td>\n",
+       "      <td>0xcc9c26a86dd55aa04dcb0066c9b8fca2983f407d</td>\n",
+       "      <td>1727136000</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>2024-09-24 02:00:00</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>21130</th>\n",
+       "      <td>1.0</td>\n",
+       "      <td>0xd1bd18d7601d106639f922f1b5d2eda025c26be7</td>\n",
+       "      <td>1727136000</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>2024-09-24 02:00:00</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7494</th>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0x4eba0ec2464ec7c746e8872078165c8ad52d346f</td>\n",
+       "      <td>1727136000</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>2024-09-24 02:00:00</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9911</th>\n",
+       "      <td>1.0</td>\n",
+       "      <td>0x61065f131e2ec851c40765bb0b078a318a36f53e</td>\n",
+       "      <td>1727136000</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>2024-09-24 02:00:00</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>26182</th>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0x7e191324f0efb8aa20b8c702d95e812e55b4179c</td>\n",
+       "      <td>1727136000</td>\n",
+       "      <td>pearl</td>\n",
+       "      <td>2024-09-24 02:00:00</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "       currentAnswer                                          id  \\\n",
+       "20792            1.0  0xcc9c26a86dd55aa04dcb0066c9b8fca2983f407d   \n",
+       "21130            1.0  0xd1bd18d7601d106639f922f1b5d2eda025c26be7   \n",
+       "7494             0.0  0x4eba0ec2464ec7c746e8872078165c8ad52d346f   \n",
+       "9911             1.0  0x61065f131e2ec851c40765bb0b078a318a36f53e   \n",
+       "26182            0.0  0x7e191324f0efb8aa20b8c702d95e812e55b4179c   \n",
+       "\n",
+       "      openingTimestamp market_creator    opening_datetime  \n",
+       "20792       1727136000     quickstart 2024-09-24 02:00:00  \n",
+       "21130       1727136000     quickstart 2024-09-24 02:00:00  \n",
+       "7494        1727136000     quickstart 2024-09-24 02:00:00  \n",
+       "9911        1727136000     quickstart 2024-09-24 02:00:00  \n",
+       "26182       1727136000          pearl 2024-09-24 02:00:00  "
+      ]
+     },
+     "execution_count": 56,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "trade_markets.tail()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 51,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import math\n",
+    "def market_KL_divergence(market_row: pd.DataFrame) -> float:\n",
+    "    \"\"\"Function to compute the divergence based on the formula\n",
+    "    Formula in https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence\"\"\"\n",
+    "    current_answer = market_row.currentAnswer  # \"yes\", \"no\"\n",
+    "    target_prob = 1  # = 100%\n",
+    "    if current_answer == \"yes\":\n",
+    "        candidate_prob = market_row.first_outcome_prob\n",
+    "    else:  # \"no\"\n",
+    "        candidate_prob = market_row.second_outcome_prob\n",
+    "\n",
+    "    # we have only one sample, the final probability based on tokens\n",
+    "    kl_divergence = candidate_prob * round(math.log(candidate_prob / target_prob), 4)\n",
+    "    return kl_divergence"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "719"
+      ]
+     },
+     "execution_count": 36,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(trade_markets)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "719"
+      ]
+     },
+     "execution_count": 37,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(list(trade_markets.id.unique()))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "currentAnswer\n",
+       "0x0000000000000000000000000000000000000000000000000000000000000001    407\n",
+       "0x0000000000000000000000000000000000000000000000000000000000000000    241\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 38,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "trade_markets.currentAnswer.value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "INVALID_ANSWER_HEX = (\n",
+    "    \"0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "def convert_hex_to_int(x):\n",
+    "    \"\"\"Convert hex to int\"\"\"\n",
+    "    if isinstance(x, float):\n",
+    "        return np.nan\n",
+    "    if isinstance(x, str):\n",
+    "        if x == INVALID_ANSWER_HEX:\n",
+    "            return -1\n",
+    "        answer = int(x, 16)\n",
+    "        return answer\n",
+    "        "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "market_ids = list(markets.id.unique())\n",
+    "for i in range(len(trade_markets)):\n",
+    "    market = trade_markets.iloc[i]\n",
+    "    if market.id in market_ids:\n",
+    "        current_answer = convert_hex_to_int(market.currentAnswer)\n",
+    "        market_answer = markets.loc[markets[\"id\"]==market.id].currentAnswer.values[0]\n",
+    "        print(f\"current answer = {current_answer} and market answer {market_answer}\")\n",
+    "        trade_markets.at[i, \"currentAnswer\"] = market_answer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "markets[\"currentAnswer\"] = markets[\"currentAnswer\"].apply(lambda x: convert_hex_to_int(x))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "currentAnswer\n",
+       " 1.0    407\n",
+       " 0.0    241\n",
+       "-1.0     84\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "markets.currentAnswer.value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 70,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0.0769610411361284"
+      ]
+     },
+     "execution_count": 70,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import math\n",
+    "\n",
+    "candidate_prob = 9/25\n",
+    "target_prob = 1/3\n",
+    "math.log(candidate_prob/target_prob)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 72,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "KL divergence: 6.296890976997244\n"
+     ]
+    }
+   ],
+   "source": [
+    "import numpy as np\n",
+    "\n",
+    "def kl_divergence(p, q):\n",
+    "    \"\"\"\n",
+    "    Compute KL divergence for a single sample with two probabilities.\n",
+    "    \n",
+    "    :param p: First probability (true distribution)\n",
+    "    :param q: Second probability (approximating distribution)\n",
+    "    :return: KL divergence value\n",
+    "    \"\"\"\n",
+    "    # Ensure probabilities sum to 1\n",
+    "    p = np.array([p, 1-p])\n",
+    "    q = np.array([q, 1-q])\n",
+    "    \n",
+    "    # Avoid division by zero\n",
+    "    epsilon = 1e-10\n",
+    "    q = np.clip(q, epsilon, 1-epsilon)\n",
+    "    \n",
+    "    # Compute KL divergence\n",
+    "    kl_div = np.sum(p * np.log(p / q))\n",
+    "    \n",
+    "    return kl_div\n",
+    "\n",
+    "# Example usage\n",
+    "p = 0.7  # probability from true distribution\n",
+    "q = 1.0  # probability from approximating distribution\n",
+    "\n",
+    "result = kl_divergence(p, q)\n",
+    "print(f\"KL divergence: {result}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 74,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "KL divergence: inf\n"
+     ]
+    }
+   ],
+   "source": [
+    "from scipy.special import kl_div\n",
+    "\n",
+    "# For multiple probabilities\n",
+    "p = np.array([0.3, 0.7])\n",
+    "q = np.array([0.0, 1.0])\n",
+    "\n",
+    "kl = np.sum(kl_div(p, q))\n",
+    "print(f\"KL divergence: {kl}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "This library is not useful if we have extreme values"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 75,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>currentAnswer</th>\n",
+       "      <th>id</th>\n",
+       "      <th>openingTimestamp</th>\n",
+       "      <th>market_creator</th>\n",
+       "      <th>opening_datetime</th>\n",
+       "      <th>first_outcome_prob</th>\n",
+       "      <th>second_outcome_prob</th>\n",
+       "      <th>kl_divergence</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>yes</td>\n",
+       "      <td>0x67490193504b49a247d6a3ba7d441e9894d9615f</td>\n",
+       "      <td>1722470400</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>2024-08-01 02:00:00</td>\n",
+       "      <td>0.8145</td>\n",
+       "      <td>0.1855</td>\n",
+       "      <td>3.791664</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>no</td>\n",
+       "      <td>0x17f2c97bf52a79671878201bf2995a3b6daba041</td>\n",
+       "      <td>1722470400</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>2024-08-01 02:00:00</td>\n",
+       "      <td>0.1975</td>\n",
+       "      <td>0.8025</td>\n",
+       "      <td>4.050688</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>no</td>\n",
+       "      <td>0xbca6aa704a02a5c5a766ff829dacc81aee5547cf</td>\n",
+       "      <td>1722470400</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>2024-08-01 02:00:00</td>\n",
+       "      <td>0.6969</td>\n",
+       "      <td>0.3031</td>\n",
+       "      <td>15.433247</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>no</td>\n",
+       "      <td>0x221c71bab604691b0b8805c1c433fc8e22123a67</td>\n",
+       "      <td>1722470400</td>\n",
+       "      <td>pearl</td>\n",
+       "      <td>2024-08-01 02:00:00</td>\n",
+       "      <td>0.4757</td>\n",
+       "      <td>0.5243</td>\n",
+       "      <td>10.261432</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>no</td>\n",
+       "      <td>0xe4d078b9be12319c0063f58dc10f19604a5df163</td>\n",
+       "      <td>1722470400</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>2024-08-01 02:00:00</td>\n",
+       "      <td>0.3473</td>\n",
+       "      <td>0.6527</td>\n",
+       "      <td>7.351119</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "  currentAnswer                                          id openingTimestamp  \\\n",
+       "0           yes  0x67490193504b49a247d6a3ba7d441e9894d9615f       1722470400   \n",
+       "1            no  0x17f2c97bf52a79671878201bf2995a3b6daba041       1722470400   \n",
+       "2            no  0xbca6aa704a02a5c5a766ff829dacc81aee5547cf       1722470400   \n",
+       "3            no  0x221c71bab604691b0b8805c1c433fc8e22123a67       1722470400   \n",
+       "4            no  0xe4d078b9be12319c0063f58dc10f19604a5df163       1722470400   \n",
+       "\n",
+       "  market_creator    opening_datetime  first_outcome_prob  second_outcome_prob  \\\n",
+       "0     quickstart 2024-08-01 02:00:00              0.8145               0.1855   \n",
+       "1     quickstart 2024-08-01 02:00:00              0.1975               0.8025   \n",
+       "2     quickstart 2024-08-01 02:00:00              0.6969               0.3031   \n",
+       "3          pearl 2024-08-01 02:00:00              0.4757               0.5243   \n",
+       "4     quickstart 2024-08-01 02:00:00              0.3473               0.6527   \n",
+       "\n",
+       "   kl_divergence  \n",
+       "0       3.791664  \n",
+       "1       4.050688  \n",
+       "2      15.433247  \n",
+       "3      10.261432  \n",
+       "4       7.351119  "
+      ]
+     },
+     "execution_count": 75,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "markets_div = pd.read_parquet(\"../data/closed_markets_div.parquet\")\n",
+    "markets_div.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 76,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>currentAnswer</th>\n",
+       "      <th>id</th>\n",
+       "      <th>openingTimestamp</th>\n",
+       "      <th>market_creator</th>\n",
+       "      <th>opening_datetime</th>\n",
+       "      <th>first_outcome_prob</th>\n",
+       "      <th>second_outcome_prob</th>\n",
+       "      <th>kl_divergence</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>642</th>\n",
+       "      <td>yes</td>\n",
+       "      <td>0x4eba0ec2464ec7c746e8872078165c8ad52d346f</td>\n",
+       "      <td>1727136000</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>2024-09-24 02:00:00</td>\n",
+       "      <td>0.5392</td>\n",
+       "      <td>0.4608</td>\n",
+       "      <td>9.920241</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>643</th>\n",
+       "      <td>no</td>\n",
+       "      <td>0x3535b4cea3ea7b1862fbe1af5a458702cc1c0dad</td>\n",
+       "      <td>1727136000</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>2024-09-24 02:00:00</td>\n",
+       "      <td>0.2812</td>\n",
+       "      <td>0.7188</td>\n",
+       "      <td>5.880786</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>644</th>\n",
+       "      <td>yes</td>\n",
+       "      <td>0x7e191324f0efb8aa20b8c702d95e812e55b4179c</td>\n",
+       "      <td>1727136000</td>\n",
+       "      <td>pearl</td>\n",
+       "      <td>2024-09-24 02:00:00</td>\n",
+       "      <td>0.5000</td>\n",
+       "      <td>0.5000</td>\n",
+       "      <td>10.819778</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>645</th>\n",
+       "      <td>no</td>\n",
+       "      <td>0xd1bd18d7601d106639f922f1b5d2eda025c26be7</td>\n",
+       "      <td>1727136000</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>2024-09-24 02:00:00</td>\n",
+       "      <td>0.5000</td>\n",
+       "      <td>0.5000</td>\n",
+       "      <td>10.819778</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>646</th>\n",
+       "      <td>no</td>\n",
+       "      <td>0x61065f131e2ec851c40765bb0b078a318a36f53e</td>\n",
+       "      <td>1727136000</td>\n",
+       "      <td>quickstart</td>\n",
+       "      <td>2024-09-24 02:00:00</td>\n",
+       "      <td>0.5000</td>\n",
+       "      <td>0.5000</td>\n",
+       "      <td>10.819778</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "    currentAnswer                                          id  \\\n",
+       "642           yes  0x4eba0ec2464ec7c746e8872078165c8ad52d346f   \n",
+       "643            no  0x3535b4cea3ea7b1862fbe1af5a458702cc1c0dad   \n",
+       "644           yes  0x7e191324f0efb8aa20b8c702d95e812e55b4179c   \n",
+       "645            no  0xd1bd18d7601d106639f922f1b5d2eda025c26be7   \n",
+       "646            no  0x61065f131e2ec851c40765bb0b078a318a36f53e   \n",
+       "\n",
+       "    openingTimestamp market_creator    opening_datetime  first_outcome_prob  \\\n",
+       "642       1727136000     quickstart 2024-09-24 02:00:00              0.5392   \n",
+       "643       1727136000     quickstart 2024-09-24 02:00:00              0.2812   \n",
+       "644       1727136000          pearl 2024-09-24 02:00:00              0.5000   \n",
+       "645       1727136000     quickstart 2024-09-24 02:00:00              0.5000   \n",
+       "646       1727136000     quickstart 2024-09-24 02:00:00              0.5000   \n",
+       "\n",
+       "     second_outcome_prob  kl_divergence  \n",
+       "642               0.4608       9.920241  \n",
+       "643               0.7188       5.880786  \n",
+       "644               0.5000      10.819778  \n",
+       "645               0.5000      10.819778  \n",
+       "646               0.5000      10.819778  "
+      ]
+     },
+     "execution_count": 76,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "markets_div.tail()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 77,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "647"
+      ]
+     },
+     "execution_count": 77,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(markets_div)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "hf_dashboards",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

scripts/closed_markets_divergence.py

@@ -0,0 +1,252 @@
+from pathlib import Path
+import os
+import math
+import pandas as pd
+import numpy as np
+from typing import Any, Union
+from string import Template
+import requests
+import pickle
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from tqdm import tqdm
+import time
+from datetime import datetime
+
+NUM_WORKERS = 10
+IPFS_POLL_INTERVAL = 0.07
+INVALID_ANSWER_HEX = (
+    "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
+)
+INVALID_ANSWER = -1
+SCRIPTS_DIR = Path(__file__).parent
+ROOT_DIR = SCRIPTS_DIR.parent
+DATA_DIR = ROOT_DIR / "data"
+SUBGRAPH_API_KEY = os.environ.get("SUBGRAPH_API_KEY", None)
+OMEN_SUBGRAPH_URL = Template(
+    """https://gateway-arbitrum.network.thegraph.com/api/${subgraph_api_key}/subgraphs/id/9fUVQpFwzpdWS9bq5WkAnmKbNNcoBwatMR4yZq81pbbz"""
+)
+get_token_amounts_query = Template(
+    """
+    {
+  
+      fpmmLiquidities(
+        where: {
+                fpmm_: {
+                        creator: "${fpmm_creator}",
+                        id: "${fpmm_id}",
+                    },
+                id_gt: ""
+                }
+                orderBy: creationTimestamp
+                orderDirection: asc
+        )
+      {
+        id
+        outcomeTokenAmounts
+        creationTimestamp
+        additionalLiquidityParameter
+      }
+    }
+"""
+)
+CREATOR = "0x89c5cc945dd550BcFfb72Fe42BfF002429F46Fec"
+PEARL_CREATOR = "0xFfc8029154ECD55ABED15BD428bA596E7D23f557"
+market_creators_map = {"quickstart": CREATOR, "pearl": PEARL_CREATOR}
+headers = {
+    "Accept": "application/json, multipart/mixed",
+    "Content-Type": "application/json",
+}
+
+
+def _to_content(q: str) -> dict[str, Any]:
+    """Convert the given query string to payload content, i.e., add it under a `queries` key and convert it to bytes."""
+    finalized_query = {
+        "query": q,
+        "variables": None,
+        "extensions": {"headers": None},
+    }
+    return finalized_query
+
+
+def collect_liquidity_info(
+    index: int, fpmm_id: str, market_creator: str
+) -> dict[str, Any]:
+    omen_subgraph = OMEN_SUBGRAPH_URL.substitute(subgraph_api_key=SUBGRAPH_API_KEY)
+    market_creator_id = market_creators_map[market_creator]
+    query = get_token_amounts_query.substitute(
+        fpmm_creator=market_creator_id.lower(),
+        fpmm_id=fpmm_id,
+    )
+    content_json = _to_content(query)
+    # print(f"Executing liquidity query {query}")
+    res = requests.post(omen_subgraph, headers=headers, json=content_json)
+    result_json = res.json()
+    tokens_info = result_json.get("data", {}).get("fpmmLiquidities", [])
+    if not tokens_info:
+        return None
+
+    # the second item is the final information of the market
+    first_info = tokens_info[1]
+    token_amounts = [int(x) for x in first_info["outcomeTokenAmounts"]]
+    time.sleep(IPFS_POLL_INTERVAL)
+    return {fpmm_id: token_amounts}
+
+
+def convert_hex_to_int(x: Union[str, float]) -> Union[int, float]:
+    """Convert hex to int"""
+    if isinstance(x, float):
+        return np.nan
+    if isinstance(x, str):
+        if x == INVALID_ANSWER_HEX:
+            return "invalid"
+        return "yes" if int(x, 16) == 0 else "no"
+
+
+def get_closed_markets():
+    print("Reading parquet file with closed markets data from trades")
+    try:
+        markets = pd.read_parquet(DATA_DIR / "fpmmTrades.parquet")
+    except Exception:
+        print("Error reading the parquet file")
+
+    columns_of_interest = [
+        "fpmm.currentAnswer",
+        "fpmm.id",
+        "fpmm.openingTimestamp",
+        "market_creator",
+    ]
+    markets = markets[columns_of_interest]
+    markets.rename(
+        columns={
+            "fpmm.currentAnswer": "currentAnswer",
+            "fpmm.openingTimestamp": "openingTimestamp",
+            "fpmm.id": "id",
+        },
+        inplace=True,
+    )
+    markets = markets.drop_duplicates(subset=["id"], keep="last")
+    # remove invalid answers
+    markets = markets.loc[markets["currentAnswer"] != INVALID_ANSWER_HEX]
+    markets["currentAnswer"] = markets["currentAnswer"].apply(
+        lambda x: convert_hex_to_int(x)
+    )
+    markets.dropna(inplace=True)
+    markets["opening_datetime"] = markets["openingTimestamp"].apply(
+        lambda x: datetime.fromtimestamp(int(x))
+    )
+    markets = markets.sort_values(by="opening_datetime", ascending=True)
+    return markets
+
+
+def kl_divergence(p, q):
+    """
+    Compute KL divergence for a single sample with two probabilities.
+
+    :param p: First probability (true distribution)
+    :param q: Second probability (approximating distribution)
+    :return: KL divergence value
+    """
+    # Ensure probabilities sum to 1
+    p = np.array([p, 1 - p])
+    q = np.array([q, 1 - q])
+
+    # Avoid division by zero
+    epsilon = 1e-10
+    q = np.clip(q, epsilon, 1 - epsilon)
+
+    # Compute KL divergence
+    kl_div = np.sum(p * np.log(p / q))
+
+    return kl_div
+
+
+def market_KL_divergence(market_row: pd.DataFrame) -> float:
+    """Function to compute the divergence based on the formula
+    Formula in https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence"""
+    current_answer = market_row.currentAnswer  # "yes", "no"
+    candidate_prob = market_row.first_outcome_prob
+    target_prob = 1.0  # for yes outcome
+    if current_answer == "no":
+        target_prob = 0.0  # = 0% for yes outcome
+
+    # we have only one sample, the final probability based on tokens
+    return kl_divergence(candidate_prob, target_prob)
+
+
+def compute_tokens_prob(token_amounts: list) -> list:
+    first_token_amounts = token_amounts[0]
+    second_token_amounts = token_amounts[1]
+    total_tokens = first_token_amounts + second_token_amounts
+    first_token_prob = 1 - round((first_token_amounts / total_tokens), 4)
+    return [first_token_prob, 1 - first_token_prob]
+
+
+def prepare_closed_markets_data():
+    closed_markets = get_closed_markets()
+    closed_markets["first_outcome_prob"] = -1.0
+    closed_markets["second_outcome_prob"] = -1.0
+    total_markets = len(closed_markets)
+    markets_no_info = []
+    no_info = 0
+    with ThreadPoolExecutor(max_workers=NUM_WORKERS) as executor:
+        futures = []
+        for i in range(total_markets):
+            futures.append(
+                executor.submit(
+                    collect_liquidity_info,
+                    i,
+                    closed_markets.iloc[i].id,
+                    closed_markets.iloc[i].market_creator,
+                )
+            )
+        markets_with_info = 0
+        for future in tqdm(
+            as_completed(futures),
+            total=len(futures),
+            desc=f"Fetching Market liquidity info",
+        ):
+            token_amounts_dict = future.result()
+            if token_amounts_dict:
+                fpmm_id, token_amounts = token_amounts_dict.popitem()
+                if token_amounts:
+                    tokens_prob = compute_tokens_prob(token_amounts)
+                    closed_markets.loc[
+                        closed_markets["id"] == fpmm_id, "first_outcome_prob"
+                    ] = tokens_prob[0]
+                    closed_markets.loc[
+                        closed_markets["id"] == fpmm_id, "second_outcome_prob"
+                    ] = tokens_prob[1]
+                    markets_with_info += 1
+                else:
+                    tqdm.write(f"Skipping market with no liquidity info")
+                    markets_no_info.append(i)
+            else:
+                tqdm.write(f"Skipping market with no liquidity info")
+                no_info += 1
+
+    print(f"Markets with info = {markets_with_info}")
+    # Removing markets with no liq info
+    closed_markets = closed_markets.loc[closed_markets["first_outcome_prob"] != -1.0]
+    print(
+        f"Finished computing all markets liquidity info. Final length = {len(closed_markets)}"
+    )
+    if len(markets_no_info) > 0:
+        print(
+            f"There were {len(markets_no_info)} markets with no liquidity info. Printing some index of the dataframe"
+        )
+        with open("no_liq_info.pickle", "wb") as file:
+            pickle.dump(markets_no_info, file)
+        print(markets_no_info[:1])
+    print(closed_markets.head())
+    # Add the Kullback–Leibler divergence values
+    print("Computing Kullback–Leibler (KL) divergence")
+    closed_markets["kl_divergence"] = closed_markets.apply(
+        lambda x: market_KL_divergence(x), axis=1
+    )
+    closed_markets.to_parquet(DATA_DIR / "closed_markets_div.parquet", index=False)
+    print("Finished preparing final dataset for visualization")
+    print(closed_markets.head())
+
+
+if __name__ == "__main__":
+    prepare_closed_markets_data()

scripts/metrics.py

@@ -69,9 +69,9 @@ def compute_trader_metrics_by_market_creator(
         if len(filtered_traders_data) == 0:
             tqdm.write(f"No data. Skipping market creator {market_creator}")
             return {}  # No Data
-    tqdm.write(
-        f"Volume of data for trader {trader_address} and market creator {market_creator} = {len(filtered_traders_data)}"
-    )
+    # tqdm.write(
+    #     f"Volume of data for trader {trader_address} and market creator {market_creator} = {len(filtered_traders_data)}"
+    # )
     metrics = compute_metrics(trader_address, filtered_traders_data)
     return metrics
 

tabs/market_plots.py

@@ -0,0 +1,37 @@
+import pandas as pd
+import gradio as gr
+import matplotlib.pyplot as plt
+import seaborn as sns
+from typing import Tuple
+import plotly.express as px
+
+
+def plot_kl_div_per_market(closed_markets: pd.DataFrame) -> gr.Plot:
+
+    # adding the total
+    all_markets = closed_markets.copy(deep=True)
+    all_markets["market_creator"] = "all"
+
+    # merging both dataframes
+    final_markets = pd.concat([closed_markets, all_markets], ignore_index=True)
+    final_markets = final_markets.sort_values(by="opening_datetime", ascending=True)
+
+    fig = px.box(
+        final_markets,
+        x="month_year_week",
+        y="kl_divergence",
+        color="market_creator",
+        color_discrete_sequence=["purple", "goldenrod", "darkgreen"],
+        category_orders={"market_creator": ["pearl", "quickstart", "all"]},
+    )
+    fig.update_traces(boxmean=True)
+    fig.update_layout(
+        xaxis_title="Markets closing Week",
+        yaxis_title="Kullback–Leibler divergence",
+        legend=dict(yanchor="top", y=0.5),
+    )
+    fig.update_xaxes(tickformat="%b %d\n%Y")
+
+    return gr.Plot(
+        value=fig,
+    )

tabs/trader_plots.py

@@ -12,7 +12,7 @@ trader_metric_choices = [
 default_trader_metric = "ROI"
 
 
-def get_trader_metrics_text() -> gr.Markdown:
+def get_metrics_text() -> gr.Markdown:
     metric_text = """ 
         ## Description of the graph
         These metrics are computed weekly. The statistical measures are: