removing daily graphs

app.py

@@ -18,10 +18,8 @@ from tabs.staking import plot_staking_trades_per_market_by_week
 
 from tabs.metrics import (
     trade_metric_choices,
-    trade_daily_metric_choices,
     tool_metric_choices,
     default_trade_metric,
-    default_daily_metric,
     default_tool_metric,
     plot_trade_metrics,
     get_trade_metrics_text,
@@ -54,9 +52,7 @@ from tabs.error import (
     plot_tool_error_data_by_market,
 )
 
-from tabs.daily_graphs import get_current_week_data, plot_daily_metrics
 from tabs.about import about_olas_predict, about_this_dashboard
-import matplotlib.pyplot as plt
 from scripts.utils import INC_TOOLS
 
 
@@ -112,14 +108,8 @@ def get_all_data():
     Get all data from the tools.parquet, tools_accuracy and trades parquet files
     """
     logger.info("Getting all data")
-    # Query to fetch daily live data
-    con = duckdb.connect(":memory:")
-    query5 = f"""
-    SELECT *
-    FROM read_parquet('./data/daily_info.parquet')
-    """
-    df5 = con.execute(query5).fetchdf()
 
+    con = duckdb.connect(":memory:")
     # Query to fetch invalid trades data
     query4 = f"""
     SELECT *
@@ -151,16 +141,14 @@ def get_all_data():
 
     con.close()
 
-    return df1, df2, df3, df4, df5
+    return df1, df2, df3, df4
 
 
 def prepare_data():
     """
     Prepare the data for the dashboard
     """
-    tools_df, trades_df, tools_accuracy_info, invalid_trades, daily_info = (
-        get_all_data()
-    )
+    tools_df, trades_df, tools_accuracy_info, invalid_trades = get_all_data()
     print(trades_df.info())
 
     tools_df = prepare_tools(tools_df)
@@ -174,17 +162,17 @@ def prepare_data():
         invalid_trades["creation_timestamp"]
     )
     invalid_trades["creation_date"] = invalid_trades["creation_timestamp"].dt.date
-    daily_info["creation_date"] = daily_info["creation_timestamp"].dt.date
+
     # discovering outliers for ROI
     outliers = trades_df.loc[trades_df["roi"] >= 1000]
     if len(outliers) > 0:
         outliers.to_parquet("./data/outliers.parquet")
         trades_df = trades_df.loc[trades_df["roi"] < 1000]
 
-    return tools_df, trades_df, tools_accuracy_info, invalid_trades, daily_info
+    return tools_df, trades_df, tools_accuracy_info, invalid_trades
 
 
-tools_df, trades_df, tools_accuracy_info, invalid_trades, daily_info = prepare_data()
+tools_df, trades_df, tools_accuracy_info, invalid_trades = prepare_data()
 
 
 demo = gr.Blocks()
@@ -334,67 +322,6 @@ with demo:
                 inputs=[trade_na_details_selector, na_trade_details_plot],
                 outputs=[na_trade_details_plot],
             )
-        with gr.TabItem("📅 Daily trades dashboard (WIP)"):
-            current_week_trades = get_current_week_data(trades_df=trades_df)
-            live_trades_current_week = get_current_week_data(trades_df=daily_info)
-            with gr.Row():
-                gr.Markdown("# Daily live metrics for all trades")
-            with gr.Row():
-                trade_live_details_selector = gr.Dropdown(
-                    label="Select a daily live metric",
-                    choices=trade_daily_metric_choices,
-                    value=default_daily_metric,
-                )
-
-            with gr.Row():
-                with gr.Column(scale=3):
-                    trade_live_details_plot = plot_daily_metrics(
-                        metric_name=default_daily_metric,
-                        trades_df=live_trades_current_week,
-                    )
-                with gr.Column(scale=1):
-                    trade_details_text = get_trade_metrics_text()
-
-            def update_trade_live_details(trade_detail, trade_live_details_plot):
-                new_a_plot = plot_daily_metrics(
-                    metric_name=trade_detail, trades_df=live_trades_current_week
-                )
-                return new_a_plot
-
-            trade_live_details_selector.change(
-                update_trade_live_details,
-                inputs=[trade_live_details_selector, trade_live_details_plot],
-                outputs=[trade_live_details_plot],
-            )
-
-            with gr.Row():
-                gr.Markdown("# Daily profitability metrics available for all trades")
-            with gr.Row():
-                trade_daily_details_selector = gr.Dropdown(
-                    label="Select a daily trade metric",
-                    choices=trade_metric_choices,
-                    value=default_trade_metric,
-                )
-
-            with gr.Row():
-                with gr.Column(scale=3):
-                    trade_daily_details_plot = plot_daily_metrics(
-                        metric_name=default_trade_metric, trades_df=current_week_trades
-                    )
-                with gr.Column(scale=1):
-                    trade_details_text = get_trade_metrics_text()
-
-            def update_trade_daily_details(trade_detail, trade_daily_details_plot):
-                new_a_plot = plot_daily_metrics(
-                    metric_name=trade_detail, trades_df=current_week_trades
-                )
-                return new_a_plot
-
-            trade_daily_details_selector.change(
-                update_trade_daily_details,
-                inputs=[trade_daily_details_selector, trade_daily_details_plot],
-                outputs=[trade_daily_details_plot],
-            )
 
         with gr.TabItem("🔒 Staking traders"):
             with gr.Row():

notebooks/daily_data.ipynb

@@ -1,430 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import pandas as pd"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "all_trades = pd.read_parquet('../data/all_trades_profitability.parquet')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "Timestamp('2024-11-23 01:38:25+0000', tz='UTC')"
-      ]
-     },
-     "execution_count": 3,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "max(all_trades.creation_timestamp)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "Timestamp('2024-09-22 00:02:05+0000', tz='UTC')"
-      ]
-     },
-     "execution_count": 4,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "min(all_trades.creation_timestamp)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "new_trades = pd.read_parquet('../data/new_fpmmTrades.parquet')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "<class 'pandas.core.frame.DataFrame'>\n",
-      "RangeIndex: 3798 entries, 0 to 3797\n",
-      "Data columns (total 24 columns):\n",
-      " #   Column                         Non-Null Count  Dtype \n",
-      "---  ------                         --------------  ----- \n",
-      " 0   collateralAmount               3798 non-null   object\n",
-      " 1   collateralAmountUSD            3798 non-null   object\n",
-      " 2   collateralToken                3798 non-null   object\n",
-      " 3   creationTimestamp              3798 non-null   object\n",
-      " 4   trader_address                 3798 non-null   object\n",
-      " 5   feeAmount                      3798 non-null   object\n",
-      " 6   id                             3798 non-null   object\n",
-      " 7   oldOutcomeTokenMarginalPrice   3798 non-null   object\n",
-      " 8   outcomeIndex                   3798 non-null   object\n",
-      " 9   outcomeTokenMarginalPrice      3798 non-null   object\n",
-      " 10  outcomeTokensTraded            3798 non-null   object\n",
-      " 11  title                          3798 non-null   object\n",
-      " 12  transactionHash                3798 non-null   object\n",
-      " 13  type                           3798 non-null   object\n",
-      " 14  market_creator                 3798 non-null   object\n",
-      " 15  fpmm.answerFinalizedTimestamp  0 non-null      object\n",
-      " 16  fpmm.arbitrationOccurred       3798 non-null   bool  \n",
-      " 17  fpmm.currentAnswer             0 non-null      object\n",
-      " 18  fpmm.id                        3798 non-null   object\n",
-      " 19  fpmm.isPendingArbitration      3798 non-null   bool  \n",
-      " 20  fpmm.openingTimestamp          3798 non-null   object\n",
-      " 21  fpmm.outcomes                  3798 non-null   object\n",
-      " 22  fpmm.title                     3798 non-null   object\n",
-      " 23  fpmm.condition.id              3798 non-null   object\n",
-      "dtypes: bool(2), object(22)\n",
-      "memory usage: 660.3+ KB\n"
-     ]
-    }
-   ],
-   "source": [
-    "new_trades.info()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 12,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "3798"
-      ]
-     },
-     "execution_count": 12,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "len(new_trades.id.unique())"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "Index(['collateralAmount', 'collateralAmountUSD', 'collateralToken',\n",
-       "       'creationTimestamp', 'trader_address', 'feeAmount', 'id',\n",
-       "       'oldOutcomeTokenMarginalPrice', 'outcomeIndex',\n",
-       "       'outcomeTokenMarginalPrice', 'outcomeTokensTraded', 'title',\n",
-       "       'transactionHash', 'type', 'market_creator',\n",
-       "       'fpmm.answerFinalizedTimestamp', 'fpmm.arbitrationOccurred',\n",
-       "       'fpmm.currentAnswer', 'fpmm.id', 'fpmm.isPendingArbitration',\n",
-       "       'fpmm.openingTimestamp', 'fpmm.outcomes', 'fpmm.title',\n",
-       "       'fpmm.condition.id'],\n",
-       "      dtype='object')"
-      ]
-     },
-     "execution_count": 4,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "new_trades.columns"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "'1732609530'"
-      ]
-     },
-     "execution_count": 6,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "max(new_trades.creationTimestamp)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 13,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "old_trades = pd.read_parquet('../data/fpmmTrades.parquet')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 14,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "'1732609530'"
-      ]
-     },
-     "execution_count": 14,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "max(old_trades.creationTimestamp)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 25,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "all_trades_before = pd.read_parquet('../data/daily_info.parquet')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 26,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "<class 'pandas.core.frame.DataFrame'>\n",
-      "RangeIndex: 3882 entries, 0 to 3881\n",
-      "Data columns (total 21 columns):\n",
-      " #   Column                  Non-Null Count  Dtype              \n",
-      "---  ------                  --------------  -----              \n",
-      " 0   trader_address          3882 non-null   object             \n",
-      " 1   market_creator          3882 non-null   object             \n",
-      " 2   trade_id                3882 non-null   object             \n",
-      " 3   creation_timestamp      3882 non-null   datetime64[ns, UTC]\n",
-      " 4   title                   3882 non-null   object             \n",
-      " 5   market_status           3882 non-null   object             \n",
-      " 6   collateral_amount       3882 non-null   float64            \n",
-      " 7   outcome_index           3882 non-null   object             \n",
-      " 8   trade_fee_amount        3882 non-null   float64            \n",
-      " 9   outcomes_tokens_traded  3882 non-null   float64            \n",
-      " 10  current_answer          0 non-null      object             \n",
-      " 11  is_invalid              3882 non-null   bool               \n",
-      " 12  winning_trade           0 non-null      object             \n",
-      " 13  earnings                3882 non-null   float64            \n",
-      " 14  redeemed                3882 non-null   bool               \n",
-      " 15  redeemed_amount         3882 non-null   int64              \n",
-      " 16  num_mech_calls          3882 non-null   int64              \n",
-      " 17  mech_fee_amount         3882 non-null   float64            \n",
-      " 18  net_earnings            3882 non-null   float64            \n",
-      " 19  roi                     3882 non-null   float64            \n",
-      " 20  staking                 3882 non-null   object             \n",
-      "dtypes: bool(2), datetime64[ns, UTC](1), float64(7), int64(2), object(9)\n",
-      "memory usage: 583.9+ KB\n"
-     ]
-    }
-   ],
-   "source": [
-    "all_trades_before.info()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 27,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "Index(['trader_address', 'market_creator', 'trade_id', 'creation_timestamp',\n",
-       "       'title', 'market_status', 'collateral_amount', 'outcome_index',\n",
-       "       'trade_fee_amount', 'outcomes_tokens_traded', 'current_answer',\n",
-       "       'is_invalid', 'winning_trade', 'earnings', 'redeemed',\n",
-       "       'redeemed_amount', 'num_mech_calls', 'mech_fee_amount', 'net_earnings',\n",
-       "       'roi', 'staking'],\n",
-       "      dtype='object')"
-      ]
-     },
-     "execution_count": 27,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "all_trades_before.columns"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 23,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "Timestamp('2024-11-26 10:19:30+0000', tz='UTC')"
-      ]
-     },
-     "execution_count": 23,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "max(all_trades_before.creation_timestamp)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 28,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "staking\n",
-       "non_agent      2376\n",
-       "quickstart      672\n",
-       "pearl           502\n",
-       "non_staking     332\n",
-       "Name: count, dtype: int64"
-      ]
-     },
-     "execution_count": 28,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "all_trades_before.staking.value_counts()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "all_trades_df = pd.read_parquet('../json_data/all_trades_df.parquet')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "Index(['trader_address', 'market_creator', 'trade_id', 'creation_timestamp',\n",
-       "       'title', 'market_status', 'collateral_amount', 'outcome_index',\n",
-       "       'trade_fee_amount', 'outcomes_tokens_traded', 'current_answer',\n",
-       "       'is_invalid', 'winning_trade', 'earnings', 'redeemed',\n",
-       "       'redeemed_amount', 'num_mech_calls', 'mech_fee_amount', 'net_earnings',\n",
-       "       'roi', 'staking', 'nr_mech_calls'],\n",
-       "      dtype='object')"
-      ]
-     },
-     "execution_count": 8,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "all_trades_df.columns"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "Timestamp('2024-11-23 01:38:25+0000', tz='UTC')"
-      ]
-     },
-     "execution_count": 9,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "max(all_trades_df.creation_timestamp)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "hf_dashboards",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.12.2"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}

notebooks/roi_analysis.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -15,7 +15,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -25,6 +25,45 @@
     "all_trades = pd.read_parquet('../data/all_trades_profitability.parquet')"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<class 'pandas.core.frame.DataFrame'>\n",
+      "RangeIndex: 615 entries, 0 to 614\n",
+      "Data columns (total 16 columns):\n",
+      " #   Column                           Non-Null Count  Dtype  \n",
+      "---  ------                           --------------  -----  \n",
+      " 0   trader_address                   615 non-null    object \n",
+      " 1   num_trades                       615 non-null    int64  \n",
+      " 2   num_winning_trades               615 non-null    float64\n",
+      " 3   num_redeemed                     615 non-null    float64\n",
+      " 4   total_investment                 615 non-null    float64\n",
+      " 5   total_trade_fees                 615 non-null    float64\n",
+      " 6   num_mech_calls                   615 non-null    int64  \n",
+      " 7   total_mech_fees                  615 non-null    float64\n",
+      " 8   total_earnings                   615 non-null    float64\n",
+      " 9   total_redeemed_amount            615 non-null    float64\n",
+      " 10  total_net_earnings               615 non-null    float64\n",
+      " 11  total_roi                        615 non-null    float64\n",
+      " 12  mean_mech_calls_per_trade        615 non-null    float64\n",
+      " 13  mean_mech_fee_amount_per_trade   615 non-null    float64\n",
+      " 14  total_net_earnings_wo_mech_fees  615 non-null    float64\n",
+      " 15  total_roi_wo_mech_fees           615 non-null    float64\n",
+      "dtypes: float64(13), int64(2), object(1)\n",
+      "memory usage: 77.0+ KB\n"
+     ]
+    }
+   ],
+   "source": [
+    "summary_traders.info()"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 3,
@@ -34,9 +73,8 @@
      "data": {
       "text/plain": [
        "current_answer\n",
-       " 1    12373\n",
-       " 0    10879\n",
-       "-1      801\n",
+       "1    25382\n",
+       "0    15794\n",
        "Name: count, dtype: int64"
       ]
      },
@@ -51,7 +89,49 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "pandas._libs.tslibs.timestamps.Timestamp"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "type(all_trades.iloc[0].creation_timestamp)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "is_invalid\n",
+       "False    41176\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "all_trades.is_invalid.value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
@@ -59,32 +139,33 @@
      "output_type": "stream",
      "text": [
       "<class 'pandas.core.frame.DataFrame'>\n",
-      "RangeIndex: 4872 entries, 0 to 4871\n",
-      "Data columns (total 20 columns):\n",
+      "RangeIndex: 41176 entries, 0 to 41175\n",
+      "Data columns (total 21 columns):\n",
       " #   Column                  Non-Null Count  Dtype              \n",
       "---  ------                  --------------  -----              \n",
-      " 0   trader_address          4872 non-null   object             \n",
-      " 1   market_creator          4872 non-null   object             \n",
-      " 2   trade_id                4872 non-null   object             \n",
-      " 3   creation_timestamp      4872 non-null   datetime64[ns, UTC]\n",
-      " 4   title                   4872 non-null   object             \n",
-      " 5   market_status           4872 non-null   object             \n",
-      " 6   collateral_amount       4872 non-null   float64            \n",
-      " 7   outcome_index           4872 non-null   object             \n",
-      " 8   trade_fee_amount        4872 non-null   float64            \n",
-      " 9   outcomes_tokens_traded  4872 non-null   float64            \n",
-      " 10  current_answer          4872 non-null   int64              \n",
-      " 11  is_invalid              4872 non-null   bool               \n",
-      " 12  winning_trade           4872 non-null   bool               \n",
-      " 13  earnings                4872 non-null   float64            \n",
-      " 14  redeemed                4872 non-null   bool               \n",
-      " 15  redeemed_amount         4872 non-null   float64            \n",
-      " 16  num_mech_calls          4872 non-null   int64              \n",
-      " 17  mech_fee_amount         4872 non-null   float64            \n",
-      " 18  net_earnings            4872 non-null   float64            \n",
-      " 19  roi                     4872 non-null   float64            \n",
-      "dtypes: bool(3), datetime64[ns, UTC](1), float64(8), int64(2), object(6)\n",
-      "memory usage: 661.5+ KB\n"
+      " 0   trader_address          41176 non-null  object             \n",
+      " 1   market_creator          41176 non-null  object             \n",
+      " 2   trade_id                41176 non-null  object             \n",
+      " 3   creation_timestamp      41176 non-null  datetime64[ns, UTC]\n",
+      " 4   title                   41176 non-null  object             \n",
+      " 5   market_status           41176 non-null  object             \n",
+      " 6   collateral_amount       41176 non-null  float64            \n",
+      " 7   outcome_index           41176 non-null  object             \n",
+      " 8   trade_fee_amount        41176 non-null  float64            \n",
+      " 9   outcomes_tokens_traded  41176 non-null  float64            \n",
+      " 10  current_answer          41176 non-null  int64              \n",
+      " 11  is_invalid              41176 non-null  bool               \n",
+      " 12  winning_trade           41176 non-null  bool               \n",
+      " 13  earnings                41176 non-null  float64            \n",
+      " 14  redeemed                41176 non-null  bool               \n",
+      " 15  redeemed_amount         41176 non-null  float64            \n",
+      " 16  num_mech_calls          41176 non-null  int64              \n",
+      " 17  mech_fee_amount         41176 non-null  float64            \n",
+      " 18  net_earnings            41176 non-null  float64            \n",
+      " 19  roi                     41176 non-null  float64            \n",
+      " 20  staking                 41176 non-null  object             \n",
+      "dtypes: bool(3), datetime64[ns, UTC](1), float64(8), int64(2), object(7)\n",
+      "memory usage: 5.8+ MB\n"
      ]
     }
    ],
@@ -709,7 +790,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "hf_dashboards",
    "language": "python",
    "name": "python3"
   },

notebooks/tools_accuracy.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 29,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -15,7 +15,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 41,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -24,7 +24,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 42,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
@@ -32,35 +32,35 @@
      "output_type": "stream",
      "text": [
       "<class 'pandas.core.frame.DataFrame'>\n",
-      "RangeIndex: 133835 entries, 0 to 133834\n",
+      "RangeIndex: 358454 entries, 0 to 358453\n",
       "Data columns (total 23 columns):\n",
       " #   Column                   Non-Null Count   Dtype  \n",
       "---  ------                   --------------   -----  \n",
-      " 0   request_id               133835 non-null  object \n",
-      " 1   request_block            133835 non-null  object \n",
-      " 2   prompt_request           133835 non-null  object \n",
-      " 3   tool                     133835 non-null  object \n",
-      " 4   nonce                    133835 non-null  object \n",
-      " 5   trader_address           133835 non-null  object \n",
-      " 6   deliver_block            133835 non-null  object \n",
-      " 7   error                    133835 non-null  int64  \n",
-      " 8   error_message            1873 non-null    object \n",
-      " 9   prompt_response          133664 non-null  object \n",
-      " 10  mech_address             133664 non-null  object \n",
-      " 11  p_yes                    131962 non-null  float64\n",
-      " 12  p_no                     131962 non-null  float64\n",
-      " 13  confidence               131962 non-null  float64\n",
-      " 14  info_utility             131962 non-null  float64\n",
-      " 15  vote                     104902 non-null  object \n",
-      " 16  win_probability          131962 non-null  float64\n",
-      " 17  title                    133835 non-null  object \n",
-      " 18  currentAnswer            107007 non-null  object \n",
-      " 19  request_time             133835 non-null  object \n",
-      " 20  request_month_year       133835 non-null  object \n",
-      " 21  request_month_year_week  133835 non-null  object \n",
-      " 22  market_creator           133835 non-null  object \n",
+      " 0   request_id               358454 non-null  object \n",
+      " 1   request_block            358454 non-null  object \n",
+      " 2   prompt_request           358454 non-null  object \n",
+      " 3   tool                     358454 non-null  object \n",
+      " 4   nonce                    358454 non-null  object \n",
+      " 5   trader_address           358454 non-null  object \n",
+      " 6   deliver_block            358454 non-null  object \n",
+      " 7   error                    358454 non-null  int64  \n",
+      " 8   error_message            3772 non-null    object \n",
+      " 9   prompt_response          357509 non-null  object \n",
+      " 10  mech_address             357601 non-null  object \n",
+      " 11  p_yes                    354682 non-null  float64\n",
+      " 12  p_no                     354682 non-null  float64\n",
+      " 13  confidence               354682 non-null  float64\n",
+      " 14  info_utility             354682 non-null  float64\n",
+      " 15  vote                     261707 non-null  object \n",
+      " 16  win_probability          354682 non-null  float64\n",
+      " 17  market_creator           358454 non-null  object \n",
+      " 18  title                    358454 non-null  object \n",
+      " 19  currentAnswer            287126 non-null  object \n",
+      " 20  request_time             358454 non-null  object \n",
+      " 21  request_month_year       358454 non-null  object \n",
+      " 22  request_month_year_week  358454 non-null  object \n",
       "dtypes: float64(5), int64(1), object(17)\n",
-      "memory usage: 23.5+ MB\n"
+      "memory usage: 62.9+ MB\n"
      ]
     }
    ],
@@ -68,6 +68,52 @@
     "tools.info()"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Index(['request_id', 'request_block', 'prompt_request', 'tool', 'nonce',\n",
+       "       'trader_address', 'deliver_block', 'error', 'error_message',\n",
+       "       'prompt_response', 'mech_address', 'p_yes', 'p_no', 'confidence',\n",
+       "       'info_utility', 'vote', 'win_probability', 'market_creator', 'title',\n",
+       "       'currentAnswer', 'request_time', 'request_month_year',\n",
+       "       'request_month_year_week'],\n",
+       "      dtype='object')"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tools.columns"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "str"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "type(tools.iloc[0].request_time)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 23,

notebooks/weekly_analysis.ipynb

@@ -74,13 +74,57 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
     "all_trades = pd.read_parquet('../data/all_trades_profitability.parquet')"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<class 'pandas.core.frame.DataFrame'>\n",
+      "RangeIndex: 26861 entries, 0 to 26860\n",
+      "Data columns (total 21 columns):\n",
+      " #   Column                  Non-Null Count  Dtype              \n",
+      "---  ------                  --------------  -----              \n",
+      " 0   trader_address          26861 non-null  object             \n",
+      " 1   market_creator          26861 non-null  object             \n",
+      " 2   trade_id                26861 non-null  object             \n",
+      " 3   creation_timestamp      26861 non-null  datetime64[ns, UTC]\n",
+      " 4   title                   26861 non-null  object             \n",
+      " 5   market_status           26861 non-null  object             \n",
+      " 6   collateral_amount       26861 non-null  float64            \n",
+      " 7   outcome_index           26861 non-null  object             \n",
+      " 8   trade_fee_amount        26861 non-null  float64            \n",
+      " 9   outcomes_tokens_traded  26861 non-null  float64            \n",
+      " 10  current_answer          26861 non-null  int64              \n",
+      " 11  is_invalid              26861 non-null  bool               \n",
+      " 12  winning_trade           26861 non-null  bool               \n",
+      " 13  earnings                26861 non-null  float64            \n",
+      " 14  redeemed                26861 non-null  bool               \n",
+      " 15  redeemed_amount         26861 non-null  float64            \n",
+      " 16  num_mech_calls          26861 non-null  int64              \n",
+      " 17  mech_fee_amount         26861 non-null  float64            \n",
+      " 18  net_earnings            26861 non-null  float64            \n",
+      " 19  roi                     26861 non-null  float64            \n",
+      " 20  staking                 26861 non-null  object             \n",
+      "dtypes: bool(3), datetime64[ns, UTC](1), float64(8), int64(2), object(7)\n",
+      "memory usage: 3.8+ MB\n"
+     ]
+    }
+   ],
+   "source": [
+    "all_trades.info()"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -4646,7 +4690,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.3"
+   "version": "3.12.2"
   },
   "orig_nbformat": 4
  },

scripts/daily_data.py

@@ -1,89 +0,0 @@
-import logging
-from utils import get_question, current_answer, RPC, measure_execution_time
-from markets import (
-    etl as mkt_etl,
-    DEFAULT_FILENAME as MARKETS_FILENAME,
-)
-from get_mech_info import (
-    get_mech_events_since_last_run,
-    update_json_files,
-)
-from pull_data import DATA_DIR, update_json_files, updating_timestamps
-from tools import DEFAULT_FILENAME as TOOLS_FILENAME, generate_tools_file
-from profitability import (
-    run_profitability_analysis,
-    analyse_all_traders,
-    label_trades_by_staking,
-)
-from update_tools_accuracy import compute_tools_accuracy
-import pandas as pd
-
-logging.basicConfig(level=logging.INFO)
-
-
-def prepare_live_metrics(
-    tools_filename="new_tools.parquet", trades_filename="new_fpmmTrades.parquet"
-):
-    fpmmTrades = pd.read_parquet(DATA_DIR / trades_filename)
-    tools = pd.read_parquet(DATA_DIR / tools_filename)
-    print("Analysing trades...")
-    all_trades_df = analyse_all_traders(fpmmTrades, tools, daily_info=True)
-
-    # staking label
-    label_trades_by_staking(all_trades_df)
-
-    # save into a separate file
-    all_trades_df.to_parquet(DATA_DIR / "daily_info.parquet", index=False)
-
-
-@measure_execution_time
-def daily_analysis():
-    """Run daily analysis for the FPMMS project."""
-    rpc = RPC
-    # Run markets ETL
-    logging.info("Running markets ETL")
-    mkt_etl(MARKETS_FILENAME)
-    logging.info("Markets ETL completed")
-
-    # get only new data
-    latest_timestamp = get_mech_events_since_last_run()
-    if latest_timestamp == None:
-        print("Error while getting the mech events")
-        return
-    logging.info(f"Finished generating the mech json files from {latest_timestamp}")
-
-    # Run tools ETL
-    logging.info("Generate and parse the tools content")
-    # generate only new file
-    generate_tools_file("new_tools_info.json", "new_tools.parquet")
-    logging.info("Tools ETL completed")
-
-    # add_current_answer("new_tools.parquet")
-
-    # # Run profitability analysis
-    logging.info("Computing trading metrics")
-    run_profitability_analysis(
-        rpc=rpc,
-        tools_filename="new_tools.parquet",
-        trades_filename="new_fpmmTrades.parquet",
-        # from_timestamp=int(latest_timestamp.timestamp()),
-        from_timestamp=latest_timestamp,
-        merge=True,
-    )
-    logging.info("Profitability analysis completed")
-
-    # merge new json files with old json files
-    update_json_files()
-
-    try:
-        updating_timestamps(rpc, TOOLS_FILENAME)
-    except Exception as e:
-        logging.error("Error while updating timestamps of tools")
-        print(e)
-
-    compute_tools_accuracy()
-
-
-if __name__ == "__main__":
-    daily_analysis()
-    prepare_live_metrics()

tabs/daily_graphs.py

@@ -1,215 +0,0 @@
-import pandas as pd
-import gradio as gr
-import gc
-import plotly.express as px
-from plotly.subplots import make_subplots
-import plotly.graph_objects as go
-from datetime import datetime, timedelta
-
-
-def plot_daily_trades(trades_df: pd.DataFrame) -> gr.Plot:
-    # get daily trades
-    daily_trades_count = (
-        trades_df.groupby("month_year_week").size().reset_index(name="trades")
-    )
-    daily_trades_count.columns = daily_trades_count.columns.astype(str)
-    print("WIP")
-
-
-def get_current_week_data(trades_df: pd.DataFrame) -> pd.DataFrame:
-    # Get current date
-    now = datetime.now()
-
-    # Get start of the current week (Monday)
-    start_of_week = now - timedelta(days=now.weekday())
-    start_of_week = start_of_week.replace(hour=0, minute=0, second=0, microsecond=0)
-    print(f"start of the week = {start_of_week}")
-
-    # Get end of the current week (Sunday)
-    end_of_week = start_of_week + timedelta(days=6)
-    end_of_week = end_of_week.replace(hour=23, minute=59, second=59, microsecond=999999)
-    print(f"end of the week = {end_of_week}")
-    trades_df["creation_date"] = pd.to_datetime(trades_df["creation_date"])
-    # Filter the dataframe
-    return trades_df[
-        (trades_df["creation_date"] >= start_of_week)
-        & (trades_df["creation_date"] <= end_of_week)
-    ]
-
-
-def get_boxplot_daily_metrics(
-    column_name: str, trades_df: pd.DataFrame
-) -> pd.DataFrame:
-    trades_filtered = trades_df[
-        ["creation_timestamp", "creation_date", "market_creator", column_name]
-    ]
-    # adding the total
-    trades_filtered_all = trades_df.copy(deep=True)
-    trades_filtered_all["market_creator"] = "all"
-
-    # merging both dataframes
-    all_filtered_trades = pd.concat(
-        [trades_filtered, trades_filtered_all], ignore_index=True
-    )
-    all_filtered_trades = all_filtered_trades.sort_values(
-        by="creation_timestamp", ascending=True
-    )
-    gc.collect()
-    return all_filtered_trades
-
-
-def plot_daily_metrics(
-    metric_name: str, trades_df: pd.DataFrame, trader_filter: str = None
-) -> gr.Plot:
-    """Plots the trade metrics."""
-
-    if metric_name == "mech calls":
-        metric_name = "mech_calls"
-        column_name = "num_mech_calls"
-        yaxis_title = "Nr of mech calls per trade"
-    elif metric_name == "ROI":
-        column_name = "roi"
-        yaxis_title = "ROI (net profit/cost)"
-    elif metric_name == "collateral amount":
-        metric_name = "collateral_amount"
-        column_name = metric_name
-        yaxis_title = "Collateral amount per trade (xDAI)"
-    elif metric_name == "net earnings":
-        metric_name = "net_earnings"
-        column_name = metric_name
-        yaxis_title = "Net profit per trade (xDAI)"
-    else:  # earnings
-        column_name = metric_name
-        yaxis_title = "Gross profit per trade (xDAI)"
-
-    color_discrete = ["purple", "darkgoldenrod", "darkgreen"]
-
-    if trader_filter == "agent":
-        trades_filtered = get_boxplot_daily_metrics(
-            column_name, trades_df.loc[trades_df["staking"] != "non_agent"]
-        )
-        color_discrete = ["darkviolet", "goldenrod", "green"]
-    elif trader_filter == "non_agent":
-        trades_filtered = get_boxplot_daily_metrics(
-            column_name, trades_df.loc[trades_df["staking"] == "non_agent"]
-        )
-    else:
-        trades_filtered = get_boxplot_daily_metrics(column_name, trades_df)
-    color_mapping = [
-        "darkviolet",
-        "purple",
-        "goldenrod",
-        "darkgoldenrod",
-        "green",
-        "darkgreen",
-    ]
-
-    # Create binary staking category
-    trades_filtered["trader_type"] = trades_filtered["staking"].apply(
-        lambda x: "non_agent" if x == "non_agent" else "agent"
-    )
-    trades_filtered["trader_market"] = trades_filtered.apply(
-        lambda x: (x["trader_type"], x["market_creator"]), axis=1
-    )
-    fig = px.box(
-        trades_filtered,
-        x="creation_date",
-        y=column_name,
-        color="trader_market",
-        color_discrete_sequence=color_mapping,
-        category_orders={
-            "market_creator": ["pearl", "quickstart", "all"],
-            "trader_market": [
-                ("agent", "pearl"),
-                ("non_agent", "pearl"),
-                ("agent", "quickstart"),
-                ("non_agent", "quickstart"),
-                ("agent", "all"),
-                ("non_agent", "all"),
-            ],
-        },
-        facet_col="market_creator",
-    )
-    fig.update_traces(boxmean=True)
-    fig.update_layout(
-        xaxis_title="Day",
-        yaxis_title=yaxis_title,
-        legend=dict(yanchor="top", y=0.5),
-    )
-    for axis in fig.layout:
-        if axis.startswith("xaxis"):
-            fig.layout[axis].update(title="Day")
-    fig.update_xaxes(tickformat="%b %d")
-    return gr.Plot(
-        value=fig,
-    )
-
-
-def plot_daily_metrics_v2(
-    metric_name: str, trades_df: pd.DataFrame, trader_filter: str = None
-) -> gr.Plot:
-    """Plots the trade metrics."""
-
-    if metric_name == "mech calls":
-        metric_name = "mech_calls"
-        column_name = "num_mech_calls"
-        yaxis_title = "Nr of mech calls per trade"
-    elif metric_name == "ROI":
-        column_name = "roi"
-        yaxis_title = "ROI (net profit/cost)"
-    elif metric_name == "collateral amount":
-        metric_name = "collateral_amount"
-        column_name = metric_name
-        yaxis_title = "Collateral amount per trade (xDAI)"
-    elif metric_name == "net earnings":
-        metric_name = "net_earnings"
-        column_name = metric_name
-        yaxis_title = "Net profit per trade (xDAI)"
-    else:  # earnings
-        column_name = metric_name
-        yaxis_title = "Gross profit per trade (xDAI)"
-
-    color_discrete = ["purple", "darkgoldenrod", "darkgreen"]
-    trades_filtered = get_boxplot_daily_metrics(column_name, trades_df)
-    fig = make_subplots(rows=1, cols=2, subplot_titles=("Agent", "Non-Agents"))
-
-    # Create first boxplot for staking=True
-    fig.add_trace(
-        go.Box(
-            x=trades_filtered[trades_filtered["staking"] != "non_agent"][
-                "creation_date"
-            ],
-            y=trades_filtered[trades_filtered["staking"] != "non_agent"][column_name],
-            name="Trades from agents",
-            marker_color=color_discrete[0],
-            legendgroup="staking_true",
-            showlegend=True,
-        ),
-        row=1,
-        col=1,
-    )
-
-    # Create second boxplot for staking=False
-    fig.add_trace(
-        go.Box(
-            x=trades_filtered[trades_filtered["staking"] == False]["creation_date"],
-            y=trades_filtered[trades_filtered["staking"] == False][column_name],
-            name="Staking False",
-            marker_color=color_discrete[1],
-            legendgroup="staking_false",
-            showlegend=True,
-        ),
-        row=1,
-        col=2,
-    )
-
-    # Update layout
-    fig.update_layout(
-        height=600,
-        width=1200,
-        title_text=f"Box Plot of {column_name} by Staking Status",
-        showlegend=True,
-    )
-
-    # Update y-axes to have the same range
-    fig.update_yaxes(matches="y")

tabs/metrics.py

@@ -11,10 +11,6 @@ trade_metric_choices = [
     "ROI",
 ]
 
-trade_daily_metric_choices = [
-    "mech calls",
-    "collateral amount",
-]
 
 tool_metric_choices = {
     "Weekly Mean Mech Tool Accuracy as (Accurate Responses/All) %": "win_perc",
@@ -24,7 +20,7 @@ tool_metric_choices = {
 }
 
 default_trade_metric = "ROI"
-default_daily_metric = "collateral amount"
+
 default_tool_metric = "Weekly Mean Mech Tool Accuracy as (Accurate Responses/All) %"
 
 HEIGHT = 600