proposal of daily info graphs

app.py

@@ -18,8 +18,10 @@ from tabs.staking import plot_staking_trades_per_market_by_week
 
 from tabs.metrics import (
     trade_metric_choices,
+    trade_daily_metric_choices,
     tool_metric_choices,
     default_trade_metric,
+    default_daily_metric,
     default_tool_metric,
     plot_trade_metrics,
     get_trade_metrics_text,
@@ -51,6 +53,8 @@ from tabs.error import (
     get_error_data_overall_by_market,
     plot_tool_error_data_by_market,
 )
+
+from tabs.daily_graphs import get_current_week_data, plot_daily_metrics
 from tabs.about import about_olas_predict, about_this_dashboard
 import matplotlib.pyplot as plt
 from scripts.utils import INC_TOOLS
@@ -108,7 +112,13 @@ def get_all_data():
     Get all data from the tools.parquet, tools_accuracy and trades parquet files
     """
     logger.info("Getting all data")
+    # Query to fetch daily live data
     con = duckdb.connect(":memory:")
+    query5 = f"""
+    SELECT *
+    FROM read_parquet('./data/daily_info.parquet')
+    """
+    df5 = con.execute(query5).fetchdf()
 
     # Query to fetch invalid trades data
     query4 = f"""
@@ -141,14 +151,16 @@ def get_all_data():
 
     con.close()
 
-    return df1, df2, df3, df4
+    return df1, df2, df3, df4, df5
 
 
 def prepare_data():
     """
     Prepare the data for the dashboard
     """
-    tools_df, trades_df, tools_accuracy_info, invalid_trades = get_all_data()
+    tools_df, trades_df, tools_accuracy_info, invalid_trades, daily_info = (
+        get_all_data()
+    )
     print(trades_df.info())
 
     tools_df = prepare_tools(tools_df)
@@ -162,17 +174,17 @@ def prepare_data():
         invalid_trades["creation_timestamp"]
     )
     invalid_trades["creation_date"] = invalid_trades["creation_timestamp"].dt.date
-
+    daily_info["creation_date"] = daily_info["creation_timestamp"].dt.date
     # discovering outliers for ROI
     outliers = trades_df.loc[trades_df["roi"] >= 1000]
     if len(outliers) > 0:
         outliers.to_parquet("./data/outliers.parquet")
         trades_df = trades_df.loc[trades_df["roi"] < 1000]
 
-    return tools_df, trades_df, tools_accuracy_info, invalid_trades
+    return tools_df, trades_df, tools_accuracy_info, invalid_trades, daily_info
 
 
-tools_df, trades_df, tools_accuracy_info, invalid_trades = prepare_data()
+tools_df, trades_df, tools_accuracy_info, invalid_trades, daily_info = prepare_data()
 
 
 demo = gr.Blocks()
@@ -194,7 +206,7 @@ with demo:
     )
 
     with gr.Tabs():
-        with gr.TabItem("🔥Trades Dashboard"):
+        with gr.TabItem("🔥 Weekly Trades Dashboard"):
             with gr.Row():
                 gr.Markdown("# Trend of weekly trades")
             with gr.Row():
@@ -204,7 +216,9 @@ with demo:
 
             with gr.Row():
                 with gr.Column(scale=1):
-                    gr.Markdown("# Weekly percentage of winning for Agent based trades")
+                    gr.Markdown(
+                        "# Weekly percentage of winning for  🤖 Agent based trades"
+                    )
                     agent_winning_trades = (
                         integrated_plot_winning_trades_per_market_by_week_v2(
                             trades_df=trades_df, trader_filter="agent"
@@ -253,7 +267,7 @@ with demo:
 
             # Agentic traders graph
             with gr.Row():
-                gr.Markdown("# Weekly trading metrics for trades coming from Agents")
+                gr.Markdown("# Weekly trading metrics for trades coming from Agents 🤖")
             with gr.Row():
                 trade_a_details_selector = gr.Dropdown(
                     label="Select a trade metric",
@@ -308,7 +322,6 @@ with demo:
                     trade_details_text = get_trade_metrics_text()
 
             def update_na_trade_details(trade_detail, trade_details_plot):
-                print(f"user selected option= {trade_detail}")
                 new_a_plot = plot_trade_metrics(
                     metric_name=trade_detail,
                     trades_df=trades_df,
@@ -321,6 +334,67 @@ with demo:
                 inputs=[trade_na_details_selector, na_trade_details_plot],
                 outputs=[na_trade_details_plot],
             )
+        with gr.TabItem("📅 Daily trades dashboard (WIP)"):
+            current_week_trades = get_current_week_data(trades_df=trades_df)
+            live_trades_current_week = get_current_week_data(trades_df=daily_info)
+            with gr.Row():
+                gr.Markdown("# Daily live metrics for all trades")
+            with gr.Row():
+                trade_live_details_selector = gr.Dropdown(
+                    label="Select a daily live metric",
+                    choices=trade_daily_metric_choices,
+                    value=default_daily_metric,
+                )
+
+            with gr.Row():
+                with gr.Column(scale=3):
+                    trade_live_details_plot = plot_daily_metrics(
+                        metric_name=default_daily_metric,
+                        trades_df=live_trades_current_week,
+                    )
+                with gr.Column(scale=1):
+                    trade_details_text = get_trade_metrics_text()
+
+            def update_trade_live_details(trade_detail, trade_live_details_plot):
+                new_a_plot = plot_daily_metrics(
+                    metric_name=trade_detail, trades_df=live_trades_current_week
+                )
+                return new_a_plot
+
+            trade_live_details_selector.change(
+                update_trade_live_details,
+                inputs=[trade_live_details_selector, trade_live_details_plot],
+                outputs=[trade_live_details_plot],
+            )
+
+            with gr.Row():
+                gr.Markdown("# Daily profitability metrics available for all trades")
+            with gr.Row():
+                trade_daily_details_selector = gr.Dropdown(
+                    label="Select a daily trade metric",
+                    choices=trade_metric_choices,
+                    value=default_trade_metric,
+                )
+
+            with gr.Row():
+                with gr.Column(scale=3):
+                    trade_daily_details_plot = plot_daily_metrics(
+                        metric_name=default_trade_metric, trades_df=current_week_trades
+                    )
+                with gr.Column(scale=1):
+                    trade_details_text = get_trade_metrics_text()
+
+            def update_trade_daily_details(trade_detail, trade_daily_details_plot):
+                new_a_plot = plot_daily_metrics(
+                    metric_name=trade_detail, trades_df=current_week_trades
+                )
+                return new_a_plot
+
+            trade_daily_details_selector.change(
+                update_trade_daily_details,
+                inputs=[trade_daily_details_selector, trade_daily_details_plot],
+                outputs=[trade_daily_details_plot],
+            )
 
         with gr.TabItem("🔒 Staking traders"):
             with gr.Row():

data/daily_info.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f41fdb6fc36cf8cb28980bca049de6b4fa986a9800176e269a5259a7e744c514
+size 251792

notebooks/daily_data.ipynb

@@ -0,0 +1,430 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "all_trades = pd.read_parquet('../data/all_trades_profitability.parquet')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Timestamp('2024-11-23 01:38:25+0000', tz='UTC')"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "max(all_trades.creation_timestamp)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Timestamp('2024-09-22 00:02:05+0000', tz='UTC')"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "min(all_trades.creation_timestamp)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "new_trades = pd.read_parquet('../data/new_fpmmTrades.parquet')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<class 'pandas.core.frame.DataFrame'>\n",
+      "RangeIndex: 3798 entries, 0 to 3797\n",
+      "Data columns (total 24 columns):\n",
+      " #   Column                         Non-Null Count  Dtype \n",
+      "---  ------                         --------------  ----- \n",
+      " 0   collateralAmount               3798 non-null   object\n",
+      " 1   collateralAmountUSD            3798 non-null   object\n",
+      " 2   collateralToken                3798 non-null   object\n",
+      " 3   creationTimestamp              3798 non-null   object\n",
+      " 4   trader_address                 3798 non-null   object\n",
+      " 5   feeAmount                      3798 non-null   object\n",
+      " 6   id                             3798 non-null   object\n",
+      " 7   oldOutcomeTokenMarginalPrice   3798 non-null   object\n",
+      " 8   outcomeIndex                   3798 non-null   object\n",
+      " 9   outcomeTokenMarginalPrice      3798 non-null   object\n",
+      " 10  outcomeTokensTraded            3798 non-null   object\n",
+      " 11  title                          3798 non-null   object\n",
+      " 12  transactionHash                3798 non-null   object\n",
+      " 13  type                           3798 non-null   object\n",
+      " 14  market_creator                 3798 non-null   object\n",
+      " 15  fpmm.answerFinalizedTimestamp  0 non-null      object\n",
+      " 16  fpmm.arbitrationOccurred       3798 non-null   bool  \n",
+      " 17  fpmm.currentAnswer             0 non-null      object\n",
+      " 18  fpmm.id                        3798 non-null   object\n",
+      " 19  fpmm.isPendingArbitration      3798 non-null   bool  \n",
+      " 20  fpmm.openingTimestamp          3798 non-null   object\n",
+      " 21  fpmm.outcomes                  3798 non-null   object\n",
+      " 22  fpmm.title                     3798 non-null   object\n",
+      " 23  fpmm.condition.id              3798 non-null   object\n",
+      "dtypes: bool(2), object(22)\n",
+      "memory usage: 660.3+ KB\n"
+     ]
+    }
+   ],
+   "source": [
+    "new_trades.info()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "3798"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(new_trades.id.unique())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Index(['collateralAmount', 'collateralAmountUSD', 'collateralToken',\n",
+       "       'creationTimestamp', 'trader_address', 'feeAmount', 'id',\n",
+       "       'oldOutcomeTokenMarginalPrice', 'outcomeIndex',\n",
+       "       'outcomeTokenMarginalPrice', 'outcomeTokensTraded', 'title',\n",
+       "       'transactionHash', 'type', 'market_creator',\n",
+       "       'fpmm.answerFinalizedTimestamp', 'fpmm.arbitrationOccurred',\n",
+       "       'fpmm.currentAnswer', 'fpmm.id', 'fpmm.isPendingArbitration',\n",
+       "       'fpmm.openingTimestamp', 'fpmm.outcomes', 'fpmm.title',\n",
+       "       'fpmm.condition.id'],\n",
+       "      dtype='object')"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "new_trades.columns"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'1732609530'"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "max(new_trades.creationTimestamp)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "old_trades = pd.read_parquet('../data/fpmmTrades.parquet')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'1732609530'"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "max(old_trades.creationTimestamp)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "all_trades_before = pd.read_parquet('../data/daily_info.parquet')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<class 'pandas.core.frame.DataFrame'>\n",
+      "RangeIndex: 3882 entries, 0 to 3881\n",
+      "Data columns (total 21 columns):\n",
+      " #   Column                  Non-Null Count  Dtype              \n",
+      "---  ------                  --------------  -----              \n",
+      " 0   trader_address          3882 non-null   object             \n",
+      " 1   market_creator          3882 non-null   object             \n",
+      " 2   trade_id                3882 non-null   object             \n",
+      " 3   creation_timestamp      3882 non-null   datetime64[ns, UTC]\n",
+      " 4   title                   3882 non-null   object             \n",
+      " 5   market_status           3882 non-null   object             \n",
+      " 6   collateral_amount       3882 non-null   float64            \n",
+      " 7   outcome_index           3882 non-null   object             \n",
+      " 8   trade_fee_amount        3882 non-null   float64            \n",
+      " 9   outcomes_tokens_traded  3882 non-null   float64            \n",
+      " 10  current_answer          0 non-null      object             \n",
+      " 11  is_invalid              3882 non-null   bool               \n",
+      " 12  winning_trade           0 non-null      object             \n",
+      " 13  earnings                3882 non-null   float64            \n",
+      " 14  redeemed                3882 non-null   bool               \n",
+      " 15  redeemed_amount         3882 non-null   int64              \n",
+      " 16  num_mech_calls          3882 non-null   int64              \n",
+      " 17  mech_fee_amount         3882 non-null   float64            \n",
+      " 18  net_earnings            3882 non-null   float64            \n",
+      " 19  roi                     3882 non-null   float64            \n",
+      " 20  staking                 3882 non-null   object             \n",
+      "dtypes: bool(2), datetime64[ns, UTC](1), float64(7), int64(2), object(9)\n",
+      "memory usage: 583.9+ KB\n"
+     ]
+    }
+   ],
+   "source": [
+    "all_trades_before.info()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Index(['trader_address', 'market_creator', 'trade_id', 'creation_timestamp',\n",
+       "       'title', 'market_status', 'collateral_amount', 'outcome_index',\n",
+       "       'trade_fee_amount', 'outcomes_tokens_traded', 'current_answer',\n",
+       "       'is_invalid', 'winning_trade', 'earnings', 'redeemed',\n",
+       "       'redeemed_amount', 'num_mech_calls', 'mech_fee_amount', 'net_earnings',\n",
+       "       'roi', 'staking'],\n",
+       "      dtype='object')"
+      ]
+     },
+     "execution_count": 27,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "all_trades_before.columns"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Timestamp('2024-11-26 10:19:30+0000', tz='UTC')"
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "max(all_trades_before.creation_timestamp)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "staking\n",
+       "non_agent      2376\n",
+       "quickstart      672\n",
+       "pearl           502\n",
+       "non_staking     332\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 28,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "all_trades_before.staking.value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "all_trades_df = pd.read_parquet('../json_data/all_trades_df.parquet')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Index(['trader_address', 'market_creator', 'trade_id', 'creation_timestamp',\n",
+       "       'title', 'market_status', 'collateral_amount', 'outcome_index',\n",
+       "       'trade_fee_amount', 'outcomes_tokens_traded', 'current_answer',\n",
+       "       'is_invalid', 'winning_trade', 'earnings', 'redeemed',\n",
+       "       'redeemed_amount', 'num_mech_calls', 'mech_fee_amount', 'net_earnings',\n",
+       "       'roi', 'staking', 'nr_mech_calls'],\n",
+       "      dtype='object')"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "all_trades_df.columns"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Timestamp('2024-11-23 01:38:25+0000', tz='UTC')"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "max(all_trades_df.creation_timestamp)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "hf_dashboards",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

scripts/daily_data.py

@@ -0,0 +1,89 @@
+import logging
+from utils import get_question, current_answer, RPC, measure_execution_time
+from markets import (
+    etl as mkt_etl,
+    DEFAULT_FILENAME as MARKETS_FILENAME,
+)
+from get_mech_info import (
+    get_mech_events_since_last_run,
+    update_json_files,
+)
+from pull_data import DATA_DIR, update_json_files, updating_timestamps
+from tools import DEFAULT_FILENAME as TOOLS_FILENAME, generate_tools_file
+from profitability import (
+    run_profitability_analysis,
+    analyse_all_traders,
+    label_trades_by_staking,
+)
+from update_tools_accuracy import compute_tools_accuracy
+import pandas as pd
+
+logging.basicConfig(level=logging.INFO)
+
+
+def prepare_live_metrics(
+    tools_filename="new_tools.parquet", trades_filename="new_fpmmTrades.parquet"
+):
+    fpmmTrades = pd.read_parquet(DATA_DIR / trades_filename)
+    tools = pd.read_parquet(DATA_DIR / tools_filename)
+    print("Analysing trades...")
+    all_trades_df = analyse_all_traders(fpmmTrades, tools, daily_info=True)
+
+    # staking label
+    label_trades_by_staking(all_trades_df)
+
+    # save into a separate file
+    all_trades_df.to_parquet(DATA_DIR / "daily_info.parquet", index=False)
+
+
+@measure_execution_time
+def daily_analysis():
+    """Run daily analysis for the FPMMS project."""
+    rpc = RPC
+    # Run markets ETL
+    logging.info("Running markets ETL")
+    mkt_etl(MARKETS_FILENAME)
+    logging.info("Markets ETL completed")
+
+    # get only new data
+    latest_timestamp = get_mech_events_since_last_run()
+    if latest_timestamp == None:
+        print("Error while getting the mech events")
+        return
+    logging.info(f"Finished generating the mech json files from {latest_timestamp}")
+
+    # Run tools ETL
+    logging.info("Generate and parse the tools content")
+    # generate only new file
+    generate_tools_file("new_tools_info.json", "new_tools.parquet")
+    logging.info("Tools ETL completed")
+
+    # add_current_answer("new_tools.parquet")
+
+    # # Run profitability analysis
+    logging.info("Computing trading metrics")
+    run_profitability_analysis(
+        rpc=rpc,
+        tools_filename="new_tools.parquet",
+        trades_filename="new_fpmmTrades.parquet",
+        # from_timestamp=int(latest_timestamp.timestamp()),
+        from_timestamp=latest_timestamp,
+        merge=True,
+    )
+    logging.info("Profitability analysis completed")
+
+    # merge new json files with old json files
+    update_json_files()
+
+    try:
+        updating_timestamps(rpc, TOOLS_FILENAME)
+    except Exception as e:
+        logging.error("Error while updating timestamps of tools")
+        print(e)
+
+    compute_tools_accuracy()
+
+
+if __name__ == "__main__":
+    daily_analysis()
+    prepare_live_metrics()

scripts/get_mech_info.py

@@ -143,14 +143,12 @@ def update_fpmmTrades_parquet(trades_filename: str) -> pd.DataFrame:
     ].astype(bool)
 
     # Check for duplicates
-    print(f"Initial length before removing duplicates= {len(merge_df)}")
+    print(f"Initial length before removing duplicates in fpmmTrades= {len(merge_df)}")
 
     # Remove duplicates
     # fpmm.outcomes is a numpy array
-    merge_df = merge_df.drop_duplicates(
-        subset=[col for col in merge_df.columns if col != "fpmm.outcomes"]
-    )
-    print(f"Final length before removing duplicates= {len(merge_df)}")
+    merge_df.drop_duplicates("id", inplace=True)
+    print(f"Final length after removing duplicates in fpmmTrades= {len(merge_df)}")
 
     # save the parquet file
     merge_df.to_parquet(DATA_DIR / "fpmmTrades.parquet", index=False)
@@ -169,11 +167,11 @@ def update_all_trades_parquet(new_trades_df: pd.DataFrame) -> pd.DataFrame:
     merge_df = pd.concat([old_trades_df, new_trades_df], ignore_index=True)
 
     # Check for duplicates
-    print(f"Initial length before removing duplicates= {len(merge_df)}")
+    print(f"Initial length before removing duplicates in all_trades= {len(merge_df)}")
 
     # Remove duplicates
-    merge_df.drop_duplicates(inplace=True)
-    print(f"Final length before removing duplicates= {len(merge_df)}")
+    merge_df.drop_duplicates("trade_id", inplace=True)
+    print(f"Final length after removing duplicates in all_trades = {len(merge_df)}")
     return merge_df
 
 
@@ -193,11 +191,11 @@ def update_tools_parquet(new_tools_filename: pd.DataFrame):
     merge_df = pd.concat([old_tools_df, new_tools_df], ignore_index=True)
 
     # Check for duplicates
-    print(f"Initial length before removing duplicates= {len(merge_df)}")
+    print(f"Initial length before removing duplicates in tools= {len(merge_df)}")
 
     # Remove duplicates
     merge_df.drop_duplicates(inplace=True)
-    print(f"Final length before removing duplicates= {len(merge_df)}")
+    print(f"Final length after removing duplicates in tools= {len(merge_df)}")
 
     # save the parquet file
     merge_df.to_parquet(DATA_DIR / "tools.parquet", index=False)

scripts/profitability.py

@@ -257,7 +257,6 @@ def _is_redeemed(user_json: dict[str, Any], fpmmTrade: dict[str, Any]) -> bool:
     """Returns whether the user has redeemed the position."""
     user_positions = user_json["data"]["user"]["userPositions"]
     condition_id = fpmmTrade["fpmm.condition.id"]
-
     for position in user_positions:
         position_condition_ids = position["position"]["conditionIds"]
         balance = int(position["balance"])
@@ -372,9 +371,11 @@ def prepare_profitalibity_data(
 
 def determine_market_status(trade, current_answer):
     """Determine the market status of a trade."""
-    if current_answer is np.nan and time.time() >= int(trade["fpmm.openingTimestamp"]):
+    if (current_answer is np.nan or current_answer is None) and time.time() >= int(
+        trade["fpmm.openingTimestamp"]
+    ):
         return MarketState.PENDING
-    elif current_answer == np.nan:
+    elif current_answer is np.nan or current_answer is None:
         return MarketState.OPEN
     elif trade["fpmm.isPendingArbitration"]:
         return MarketState.ARBITRATING
@@ -384,7 +385,10 @@ def determine_market_status(trade, current_answer):
 
 
 def analyse_trader(
-    trader_address: str, fpmmTrades: pd.DataFrame, tools: pd.DataFrame
+    trader_address: str,
+    fpmmTrades: pd.DataFrame,
+    tools: pd.DataFrame,
+    daily_info: bool = False,
 ) -> pd.DataFrame:
     """Analyse a trader's trades"""
     # Filter trades and tools for the given trader
@@ -406,10 +410,12 @@ def analyse_trader(
     # Iterate over the trades
     for i, trade in tqdm(trades.iterrows(), total=len(trades), desc="Analysing trades"):
         try:
-            if not trade["fpmm.currentAnswer"]:
+            market_answer = trade["fpmm.currentAnswer"]
+            if not daily_info and not market_answer:
                 print(f"Skipping trade {i} because currentAnswer is NaN")
                 continue
             # Parsing and computing shared values
+
             creation_timestamp_utc = datetime.datetime.fromtimestamp(
                 int(trade["creationTimestamp"]), tz=datetime.timezone.utc
             )
@@ -418,25 +424,29 @@ def analyse_trader(
             outcome_tokens_traded = wei_to_unit(float(trade["outcomeTokensTraded"]))
             earnings, winner_trade = (0, False)
             redemption = _is_redeemed(user_json, trade)
-            current_answer = trade["fpmm.currentAnswer"]
+            current_answer = market_answer if market_answer else None
             market_creator = trade["market_creator"]
 
             # Determine market status
             market_status = determine_market_status(trade, current_answer)
 
             # Skip non-closed markets
-            if market_status != MarketState.CLOSED:
+            if not daily_info and market_status != MarketState.CLOSED:
                 print(
                     f"Skipping trade {i} because market is not closed. Market Status: {market_status}"
                 )
                 continue
-            current_answer = convert_hex_to_int(current_answer)
+            if current_answer is not None:
+                current_answer = convert_hex_to_int(current_answer)
 
             # Compute invalidity
             is_invalid = current_answer == INVALID_ANSWER
 
             # Compute earnings and winner trade status
-            if is_invalid:
+            if current_answer is None:
+                earnings = 0.0
+                winner_trade = None
+            elif is_invalid:
                 earnings = collateral_amount
                 winner_trade = False
             elif int(trade["outcomeIndex"]) == current_answer:
@@ -491,12 +501,15 @@ def analyse_trader(
 
         except Exception as e:
             print(f"Error processing trade {i}: {e}")
+            print(trade)
             continue
 
     return trades_df
 
 
-def analyse_all_traders(trades: pd.DataFrame, tools: pd.DataFrame) -> pd.DataFrame:
+def analyse_all_traders(
+    trades: pd.DataFrame, tools: pd.DataFrame, daily_info: bool = False
+) -> pd.DataFrame:
     """Analyse all creators."""
     all_traders = []
     for trader in tqdm(
@@ -504,7 +517,7 @@ def analyse_all_traders(trades: pd.DataFrame, tools: pd.DataFrame) -> pd.DataFra
         total=len(trades["trader_address"].unique()),
         desc="Analysing creators",
     ):
-        all_traders.append(analyse_trader(trader, trades, tools))
+        all_traders.append(analyse_trader(trader, trades, tools, daily_info))
 
     # concat all creators
     all_creators_df = pd.concat(all_traders)
@@ -573,7 +586,6 @@ def run_profitability_analysis(
         rpc, tools_filename, trades_filename, from_timestamp
     )
 
-    # all trades profitability df
     print("Analysing trades...")
     all_trades_df = analyse_all_traders(fpmmTrades, tools)
 
@@ -584,7 +596,7 @@ def run_profitability_analysis(
         all_trades_df = update_all_trades_parquet(all_trades_df)
 
     # debugging purposes
-    all_trades_df.to_parquet(JSON_DATA_DIR / "all_trades_df.parquets")
+    all_trades_df.to_parquet(JSON_DATA_DIR / "all_trades_df.parquet")
     # filter invalid markets. Condition: "is_invalid" is True
     invalid_trades = all_trades_df.loc[all_trades_df["is_invalid"] == True]
     if merge:

scripts/pull_data.py

@@ -1,5 +1,4 @@
 import logging
-import os
 import pickle
 from datetime import datetime
 from concurrent.futures import ThreadPoolExecutor
@@ -14,7 +13,7 @@ from markets import (
 )
 from tools import DEFAULT_FILENAME as TOOLS_FILENAME, generate_tools_file
 from profitability import run_profitability_analysis, DEFAULT_60_DAYS_AGO_TIMESTAMP
-from utils import get_question, current_answer, RPC
+from utils import get_question, current_answer, RPC, measure_execution_time
 from get_mech_info import (
     get_mech_events_last_60_days,
     get_mech_events_since_last_run,
@@ -120,12 +119,13 @@ def updating_timestamps(rpc: str, tools_filename: str):
     gc.collect()
 
 
+@measure_execution_time
 def only_new_weekly_analysis():
     """Run weekly analysis for the FPMMS project."""
     rpc = RPC
     # Run markets ETL
     logging.info("Running markets ETL")
-    # mkt_etl(MARKETS_FILENAME)
+    mkt_etl(MARKETS_FILENAME)
     logging.info("Markets ETL completed")
 
     # New tools ETL

tabs/daily_graphs.py

@@ -0,0 +1,215 @@
+import pandas as pd
+import gradio as gr
+import gc
+import plotly.express as px
+from plotly.subplots import make_subplots
+import plotly.graph_objects as go
+from datetime import datetime, timedelta
+
+
+def plot_daily_trades(trades_df: pd.DataFrame) -> gr.Plot:
+    # get daily trades
+    daily_trades_count = (
+        trades_df.groupby("month_year_week").size().reset_index(name="trades")
+    )
+    daily_trades_count.columns = daily_trades_count.columns.astype(str)
+    print("WIP")
+
+
+def get_current_week_data(trades_df: pd.DataFrame) -> pd.DataFrame:
+    # Get current date
+    now = datetime.now()
+
+    # Get start of the current week (Monday)
+    start_of_week = now - timedelta(days=now.weekday())
+    start_of_week = start_of_week.replace(hour=0, minute=0, second=0, microsecond=0)
+    print(f"start of the week = {start_of_week}")
+
+    # Get end of the current week (Sunday)
+    end_of_week = start_of_week + timedelta(days=6)
+    end_of_week = end_of_week.replace(hour=23, minute=59, second=59, microsecond=999999)
+    print(f"end of the week = {end_of_week}")
+    trades_df["creation_date"] = pd.to_datetime(trades_df["creation_date"])
+    # Filter the dataframe
+    return trades_df[
+        (trades_df["creation_date"] >= start_of_week)
+        & (trades_df["creation_date"] <= end_of_week)
+    ]
+
+
+def get_boxplot_daily_metrics(
+    column_name: str, trades_df: pd.DataFrame
+) -> pd.DataFrame:
+    trades_filtered = trades_df[
+        ["creation_timestamp", "creation_date", "market_creator", column_name]
+    ]
+    # adding the total
+    trades_filtered_all = trades_df.copy(deep=True)
+    trades_filtered_all["market_creator"] = "all"
+
+    # merging both dataframes
+    all_filtered_trades = pd.concat(
+        [trades_filtered, trades_filtered_all], ignore_index=True
+    )
+    all_filtered_trades = all_filtered_trades.sort_values(
+        by="creation_timestamp", ascending=True
+    )
+    gc.collect()
+    return all_filtered_trades
+
+
+def plot_daily_metrics(
+    metric_name: str, trades_df: pd.DataFrame, trader_filter: str = None
+) -> gr.Plot:
+    """Plots the trade metrics."""
+
+    if metric_name == "mech calls":
+        metric_name = "mech_calls"
+        column_name = "num_mech_calls"
+        yaxis_title = "Nr of mech calls per trade"
+    elif metric_name == "ROI":
+        column_name = "roi"
+        yaxis_title = "ROI (net profit/cost)"
+    elif metric_name == "collateral amount":
+        metric_name = "collateral_amount"
+        column_name = metric_name
+        yaxis_title = "Collateral amount per trade (xDAI)"
+    elif metric_name == "net earnings":
+        metric_name = "net_earnings"
+        column_name = metric_name
+        yaxis_title = "Net profit per trade (xDAI)"
+    else:  # earnings
+        column_name = metric_name
+        yaxis_title = "Gross profit per trade (xDAI)"
+
+    color_discrete = ["purple", "darkgoldenrod", "darkgreen"]
+
+    if trader_filter == "agent":
+        trades_filtered = get_boxplot_daily_metrics(
+            column_name, trades_df.loc[trades_df["staking"] != "non_agent"]
+        )
+        color_discrete = ["darkviolet", "goldenrod", "green"]
+    elif trader_filter == "non_agent":
+        trades_filtered = get_boxplot_daily_metrics(
+            column_name, trades_df.loc[trades_df["staking"] == "non_agent"]
+        )
+    else:
+        trades_filtered = get_boxplot_daily_metrics(column_name, trades_df)
+    color_mapping = [
+        "darkviolet",
+        "purple",
+        "goldenrod",
+        "darkgoldenrod",
+        "green",
+        "darkgreen",
+    ]
+
+    # Create binary staking category
+    trades_filtered["trader_type"] = trades_filtered["staking"].apply(
+        lambda x: "non_agent" if x == "non_agent" else "agent"
+    )
+    trades_filtered["trader_market"] = trades_filtered.apply(
+        lambda x: (x["trader_type"], x["market_creator"]), axis=1
+    )
+    fig = px.box(
+        trades_filtered,
+        x="creation_date",
+        y=column_name,
+        color="trader_market",
+        color_discrete_sequence=color_mapping,
+        category_orders={
+            "market_creator": ["pearl", "quickstart", "all"],
+            "trader_market": [
+                ("agent", "pearl"),
+                ("non_agent", "pearl"),
+                ("agent", "quickstart"),
+                ("non_agent", "quickstart"),
+                ("agent", "all"),
+                ("non_agent", "all"),
+            ],
+        },
+        facet_col="market_creator",
+    )
+    fig.update_traces(boxmean=True)
+    fig.update_layout(
+        xaxis_title="Day",
+        yaxis_title=yaxis_title,
+        legend=dict(yanchor="top", y=0.5),
+    )
+    for axis in fig.layout:
+        if axis.startswith("xaxis"):
+            fig.layout[axis].update(title="Day")
+    fig.update_xaxes(tickformat="%b %d")
+    return gr.Plot(
+        value=fig,
+    )
+
+
+def plot_daily_metrics_v2(
+    metric_name: str, trades_df: pd.DataFrame, trader_filter: str = None
+) -> gr.Plot:
+    """Plots the trade metrics."""
+
+    if metric_name == "mech calls":
+        metric_name = "mech_calls"
+        column_name = "num_mech_calls"
+        yaxis_title = "Nr of mech calls per trade"
+    elif metric_name == "ROI":
+        column_name = "roi"
+        yaxis_title = "ROI (net profit/cost)"
+    elif metric_name == "collateral amount":
+        metric_name = "collateral_amount"
+        column_name = metric_name
+        yaxis_title = "Collateral amount per trade (xDAI)"
+    elif metric_name == "net earnings":
+        metric_name = "net_earnings"
+        column_name = metric_name
+        yaxis_title = "Net profit per trade (xDAI)"
+    else:  # earnings
+        column_name = metric_name
+        yaxis_title = "Gross profit per trade (xDAI)"
+
+    color_discrete = ["purple", "darkgoldenrod", "darkgreen"]
+    trades_filtered = get_boxplot_daily_metrics(column_name, trades_df)
+    fig = make_subplots(rows=1, cols=2, subplot_titles=("Agent", "Non-Agents"))
+
+    # Create first boxplot for staking=True
+    fig.add_trace(
+        go.Box(
+            x=trades_filtered[trades_filtered["staking"] != "non_agent"][
+                "creation_date"
+            ],
+            y=trades_filtered[trades_filtered["staking"] != "non_agent"][column_name],
+            name="Trades from agents",
+            marker_color=color_discrete[0],
+            legendgroup="staking_true",
+            showlegend=True,
+        ),
+        row=1,
+        col=1,
+    )
+
+    # Create second boxplot for staking=False
+    fig.add_trace(
+        go.Box(
+            x=trades_filtered[trades_filtered["staking"] == False]["creation_date"],
+            y=trades_filtered[trades_filtered["staking"] == False][column_name],
+            name="Staking False",
+            marker_color=color_discrete[1],
+            legendgroup="staking_false",
+            showlegend=True,
+        ),
+        row=1,
+        col=2,
+    )
+
+    # Update layout
+    fig.update_layout(
+        height=600,
+        width=1200,
+        title_text=f"Box Plot of {column_name} by Staking Status",
+        showlegend=True,
+    )
+
+    # Update y-axes to have the same range
+    fig.update_yaxes(matches="y")

tabs/metrics.py

@@ -2,7 +2,6 @@ import pandas as pd
 import gradio as gr
 import plotly.express as px
 import gc
-import matplotlib.pyplot as plt
 
 trade_metric_choices = [
     "mech calls",
@@ -12,6 +11,11 @@ trade_metric_choices = [
     "ROI",
 ]
 
+trade_daily_metric_choices = [
+    "mech calls",
+    "collateral amount",
+]
+
 tool_metric_choices = {
     "Weekly Mean Mech Tool Accuracy as (Accurate Responses/All) %": "win_perc",
     "Total Weekly Inaccurate Nr of Mech Tool Responses": "losses",
@@ -20,6 +24,7 @@ tool_metric_choices = {
 }
 
 default_trade_metric = "ROI"
+default_daily_metric = "collateral amount"
 default_tool_metric = "Weekly Mean Mech Tool Accuracy as (Accurate Responses/All) %"
 
 HEIGHT = 600
@@ -142,10 +147,12 @@ def plot_trade_metrics(
         column_name = metric_name
         yaxis_title = "Gross profit per trade (xDAI)"
 
+    color_discrete = ["purple", "darkgoldenrod", "darkgreen"]
     if trader_filter == "agent":
         trades_filtered = get_boxplot_metrics(
             column_name, trades_df.loc[trades_df["staking"] != "non_agent"]
         )
+        color_discrete = ["darkviolet", "goldenrod", "green"]
     elif trader_filter == "non_agent":
         trades_filtered = get_boxplot_metrics(
             column_name, trades_df.loc[trades_df["staking"] == "non_agent"]
@@ -157,7 +164,7 @@ def plot_trade_metrics(
         x="month_year_week",
         y=column_name,
         color="market_creator",
-        color_discrete_sequence=["purple", "goldenrod", "darkgreen"],
+        color_discrete_sequence=color_discrete,
         category_orders={"market_creator": ["pearl", "quickstart", "all"]},
     )
     fig.update_traces(boxmean=True)

tabs/trades.py

@@ -12,6 +12,7 @@ WIDTH = 1100
 def prepare_trades(trades_df: pd.DataFrame) -> pd.DataFrame:
     """Prepares the trades data for analysis."""
     trades_df["creation_timestamp"] = pd.to_datetime(trades_df["creation_timestamp"])
+    trades_df["creation_date"] = trades_df["creation_timestamp"].dt.date
     trades_df["creation_timestamp"] = trades_df["creation_timestamp"].dt.tz_convert(
         "UTC"
     )