adding new dataset with weekly metrics for traders and updated ROI

app.py

@@ -1,10 +1,7 @@
 import gradio as gr
 import pandas as pd
-import gzip
-import shutil
-import os
 import logging
-from huggingface_hub import hf_hub_download
+from scripts.utils import prepare_data
 
 
 from scripts.metrics import (
@@ -18,7 +15,7 @@ from scripts.retention_metrics import (
     calculate_cohort_retention,
 )
 from tabs.trader_plots import (
-    plot_trader_metrics_by_market_creator,
+    plot_trader_metrics_by_agent_categories,
     default_trader_metric,
     trader_metric_choices,
     get_metrics_text,
@@ -66,209 +63,6 @@ def get_logger():
 
 logger = get_logger()
 
-
-def load_all_data():
-
-    # all trades profitability
-    # Download the compressed file
-    gz_filepath_trades = hf_hub_download(
-        repo_id="valory/Olas-predict-dataset",
-        filename="all_trades_profitability.parquet.gz",
-        repo_type="dataset",
-    )
-
-    parquet_filepath_trades = gz_filepath_trades.replace(".gz", "")
-    parquet_filepath_trades = parquet_filepath_trades.replace("all", "")
-
-    with gzip.open(gz_filepath_trades, "rb") as f_in:
-        with open(parquet_filepath_trades, "wb") as f_out:
-            shutil.copyfileobj(f_in, f_out)
-
-    # Now read the decompressed parquet file
-    df1 = pd.read_parquet(parquet_filepath_trades)
-
-    # closed_markets_div
-    closed_markets_df = hf_hub_download(
-        repo_id="valory/Olas-predict-dataset",
-        filename="closed_markets_div.parquet",
-        repo_type="dataset",
-    )
-    df2 = pd.read_parquet(closed_markets_df)
-
-    # daily_info
-    daily_info_df = hf_hub_download(
-        repo_id="valory/Olas-predict-dataset",
-        filename="daily_info.parquet",
-        repo_type="dataset",
-    )
-    df3 = pd.read_parquet(daily_info_df)
-
-    # unknown traders
-    unknown_df = hf_hub_download(
-        repo_id="valory/Olas-predict-dataset",
-        filename="unknown_traders.parquet",
-        repo_type="dataset",
-    )
-    df4 = pd.read_parquet(unknown_df)
-
-    # retention activity
-    gz_file_path_ret = hf_hub_download(
-        repo_id="valory/Olas-predict-dataset",
-        filename="retention_activity.parquet.gz",
-        repo_type="dataset",
-    )
-    parquet_file_path_ret = gz_file_path_ret.replace(".gz", "")
-
-    with gzip.open(gz_file_path_ret, "rb") as f_in:
-        with open(parquet_file_path_ret, "wb") as f_out:
-            shutil.copyfileobj(f_in, f_out)
-    df5 = pd.read_parquet(parquet_file_path_ret)
-    # os.remove(parquet_file_path_ret)
-
-    # active_traders.parquet
-    active_traders_df = hf_hub_download(
-        repo_id="valory/Olas-predict-dataset",
-        filename="active_traders.parquet",
-        repo_type="dataset",
-    )
-    df6 = pd.read_parquet(active_traders_df)
-
-    # weekly_mech_calls.parquet
-    all_mech_calls_df = hf_hub_download(
-        repo_id="valory/Olas-predict-dataset",
-        filename="weekly_mech_calls.parquet",
-        repo_type="dataset",
-    )
-    df7 = pd.read_parquet(all_mech_calls_df)
-
-    # daa for quickstart and pearl
-    daa_qs_df = hf_hub_download(
-        repo_id="valory/Olas-predict-dataset",
-        filename="latest_result_DAA_QS.parquet",
-        repo_type="dataset",
-    )
-    df8 = pd.read_parquet(daa_qs_df)
-
-    daa_pearl_df = hf_hub_download(
-        repo_id="valory/Olas-predict-dataset",
-        filename="latest_result_DAA_Pearl.parquet",
-        repo_type="dataset",
-    )
-    df9 = pd.read_parquet(daa_pearl_df)
-    # Read weekly_avg_roi_pearl_agents.parquet
-    weekly_avg_roi_pearl_agents = hf_hub_download(
-        repo_id="valory/Olas-predict-dataset",
-        filename="weekly_avg_roi_pearl_agents.parquet",
-        repo_type="dataset",
-    )
-    df10 = pd.read_parquet(weekly_avg_roi_pearl_agents)
-
-    # two_weeks_avg_roi_pearl_agents.parquet
-    two_weeks_avg_roi_pearl_agents = hf_hub_download(
-        repo_id="valory/Olas-predict-dataset",
-        filename="two_weeks_avg_roi_pearl_agents.parquet",
-        repo_type="dataset",
-    )
-    df11 = pd.read_parquet(two_weeks_avg_roi_pearl_agents)
-    return df1, df2, df3, df4, df5, df6, df7, df8, df9, df10, df11
-
-
-def prepare_data():
-
-    (
-        all_trades,
-        closed_markets,
-        daily_info,
-        unknown_traders,
-        retention_df,
-        active_traders,
-        all_mech_calls,
-        daa_qs_df,
-        daa_pearl_df,
-        weekly_avg_roi_pearl_agents,
-        two_weeks_avg_roi_pearl_agents,
-    ) = load_all_data()
-    all_trades["creation_timestamp"] = all_trades["creation_timestamp"].dt.tz_convert(
-        "UTC"
-    )
-    all_trades = all_trades.sort_values(by="creation_timestamp", ascending=True)
-    all_trades["creation_date"] = all_trades["creation_timestamp"].dt.date
-
-    # nr-trades variable
-    volume_trades_per_trader_and_market = (
-        all_trades.groupby(["trader_address", "title"])["roi"]
-        .count()
-        .reset_index(name="nr_trades_per_market")
-    )
-
-    traders_data = pd.merge(
-        all_trades, volume_trades_per_trader_and_market, on=["trader_address", "title"]
-    )
-    daily_info["creation_date"] = daily_info["creation_timestamp"].dt.date
-    unknown_traders["creation_date"] = unknown_traders["creation_timestamp"].dt.date
-    active_traders["creation_date"] = active_traders["creation_timestamp"].dt.date
-    # adding the trader family column
-    traders_data["trader_family"] = traders_data.apply(
-        lambda x: get_traders_family(x), axis=1
-    )
-    # print(traders_data.head())
-
-    traders_data = traders_data.sort_values(by="creation_timestamp", ascending=True)
-    unknown_traders = unknown_traders.sort_values(
-        by="creation_timestamp", ascending=True
-    )
-    traders_data["month_year_week"] = (
-        traders_data["creation_timestamp"]
-        .dt.to_period("W")
-        .dt.start_time.dt.strftime("%b-%d-%Y")
-    )
-    unknown_traders["month_year_week"] = (
-        unknown_traders["creation_timestamp"]
-        .dt.to_period("W")
-        .dt.start_time.dt.strftime("%b-%d-%Y")
-    )
-    closed_markets["month_year_week"] = (
-        closed_markets["opening_datetime"]
-        .dt.to_period("W")
-        .dt.start_time.dt.strftime("%b-%d-%Y")
-    )
-
-    # prepare the daa dataframes
-    daa_pearl_df["day"] = pd.to_datetime(
-        daa_pearl_df["day"], format="%Y-%m-%d 00:00:00.000 UTC"
-    )
-    daa_qs_df["day"] = pd.to_datetime(
-        daa_qs_df["day"], format="%Y-%m-%d 00:00:00.000 UTC"
-    )
-    daa_pearl_df["day"] = daa_pearl_df["day"].dt.tz_localize("UTC")
-    daa_qs_df["day"] = daa_qs_df["day"].dt.tz_localize("UTC")
-    daa_qs_df["tx_date"] = pd.to_datetime(daa_qs_df["day"]).dt.date
-    daa_pearl_df["tx_date"] = pd.to_datetime(daa_pearl_df["day"]).dt.date
-    daa_pearl_df["seven_day_trailing_avg"] = pd.to_numeric(
-        daa_pearl_df["seven_day_trailing_avg"], errors="coerce"
-    )
-    daa_pearl_df["seven_day_trailing_avg"] = daa_pearl_df[
-        "seven_day_trailing_avg"
-    ].round(2)
-    daa_qs_df["seven_day_trailing_avg"] = pd.to_numeric(
-        daa_qs_df["seven_day_trailing_avg"], errors="coerce"
-    )
-    daa_qs_df["seven_day_trailing_avg"] = daa_qs_df["seven_day_trailing_avg"].round(2)
-    return (
-        traders_data,
-        closed_markets,
-        daily_info,
-        unknown_traders,
-        retention_df,
-        active_traders,
-        all_mech_calls,
-        daa_qs_df,
-        daa_pearl_df,
-        weekly_avg_roi_pearl_agents,
-        two_weeks_avg_roi_pearl_agents,
-    )
-
-
 (
     traders_data,
     closed_markets,
@@ -281,6 +75,7 @@ def prepare_data():
     daa_pearl_df,
     weekly_avg_roi_pearl_agents,
     two_weeks_avg_roi_pearl_agents,
+    traders_weekly_metrics_df,
 ) = prepare_data()
 
 retention_df = prepare_retention_dataset(
@@ -290,20 +85,8 @@ print("max date of retention df")
 print(max(retention_df.creation_timestamp))
 
 demo = gr.Blocks()
-# get weekly metrics by market creator: qs, pearl or all.
-weekly_metrics_by_market_creator = compute_weekly_metrics_by_market_creator(
-    traders_data=traders_data, all_mech_calls=all_mech_calls
-)
-weekly_o_metrics_by_market_creator = compute_weekly_metrics_by_market_creator(
-    traders_data=traders_data, all_mech_calls=all_mech_calls, trader_filter="Olas"
-)
-weekly_non_olas_metrics_by_market_creator = pd.DataFrame()
-if len(traders_data.loc[traders_data["staking"] == "non_Olas"]) > 0:
-    weekly_non_olas_metrics_by_market_creator = (
-        compute_weekly_metrics_by_market_creator(
-            traders_data, all_mech_calls, trader_filter="non_Olas"
-        )
-    )
+# TODO classify traders in the weekly metrics dataframe by agent type (Pearl, QS, and all)
+
 weekly_unknown_trader_metrics_by_market_creator = None
 if len(unknown_traders) > 0:
     weekly_unknown_trader_metrics_by_market_creator = (
@@ -335,9 +118,9 @@ with demo:
     gr.Markdown("This app shows the weekly performance of the traders in Olas Predict.")
 
     with gr.Tabs():
-        with gr.TabItem("🔥 Weekly metrics"):
+        with gr.TabItem("🔥 Weekly metrics (WIP)"):
             with gr.Row():
-                gr.Markdown("# Weekly metrics of all traders")
+                gr.Markdown("# Weekly metrics for all traders")
             with gr.Row():
                 trader_details_selector = gr.Dropdown(
                     label="Select a weekly trader metric",
@@ -347,17 +130,17 @@ with demo:
 
             with gr.Row():
                 with gr.Column(scale=3):
-                    trader_markets_plot = plot_trader_metrics_by_market_creator(
+                    trader_markets_plot = plot_trader_metrics_by_agent_categories(
                         metric_name=default_trader_metric,
-                        traders_df=weekly_metrics_by_market_creator,
+                        traders_df=traders_weekly_metrics_df,
                     )
                 with gr.Column(scale=1):
                     trade_details_text = get_metrics_text(trader_type=None)
 
             def update_trader_details(trader_detail):
-                return plot_trader_metrics_by_market_creator(
+                return plot_trader_metrics_by_agent_categories(
                     metric_name=trader_detail,
-                    traders_df=weekly_metrics_by_market_creator,
+                    traders_df=traders_weekly_metrics_df,
                 )
 
             trader_details_selector.change(
@@ -366,99 +149,70 @@ with demo:
                 outputs=trader_markets_plot,
             )
 
-            with gr.Row():
-                gr.Markdown("# Weekly metrics of 🌊 Olas traders")
-            with gr.Row():
-                trader_o_details_selector = gr.Dropdown(
-                    label="Select a weekly trader metric",
-                    choices=trader_metric_choices,
-                    value=default_trader_metric,
-                )
-
-            with gr.Row():
-                with gr.Column(scale=3):
-                    o_trader_markets_plot = plot_trader_metrics_by_market_creator(
-                        metric_name=default_trader_metric,
-                        traders_df=weekly_o_metrics_by_market_creator,
-                    )
-                with gr.Column(scale=1):
-                    trade_details_text = get_metrics_text(trader_type="Olas")
-
-            def update_a_trader_details(trader_detail):
-                return plot_trader_metrics_by_market_creator(
-                    metric_name=trader_detail,
-                    traders_df=weekly_o_metrics_by_market_creator,
-                )
-
-            trader_o_details_selector.change(
-                update_a_trader_details,
-                inputs=trader_o_details_selector,
-                outputs=o_trader_markets_plot,
-            )
-            if len(weekly_non_olas_metrics_by_market_creator) > 0:
-                # Non-Olas traders graph
-                with gr.Row():
-                    gr.Markdown("# Weekly metrics of Non-Olas traders")
-                with gr.Row():
-                    trader_no_details_selector = gr.Dropdown(
-                        label="Select a weekly trader metric",
-                        choices=trader_metric_choices,
-                        value=default_trader_metric,
-                    )
+            # if len(weekly_non_olas_metrics_by_market_creator) > 0:
+            #     # Non-Olas traders graph
+            #     with gr.Row():
+            #         gr.Markdown("# Weekly metrics of Non-Olas traders")
+            # with gr.Row():
+            #     trader_no_details_selector = gr.Dropdown(
+            #         label="Select a weekly trader metric",
+            #         choices=trader_metric_choices,
+            #         value=default_trader_metric,
+            #     )
 
-                with gr.Row():
-                    with gr.Column(scale=3):
-                        trader_no_markets_plot = plot_trader_metrics_by_market_creator(
-                            metric_name=default_trader_metric,
-                            traders_df=weekly_non_olas_metrics_by_market_creator,
-                        )
-                    with gr.Column(scale=1):
-                        trade_details_text = get_metrics_text(trader_type="non_Olas")
+            # with gr.Row():
+            #     with gr.Column(scale=3):
+            #         trader_no_markets_plot = plot_trader_metrics_by_market_creator(
+            #             metric_name=default_trader_metric,
+            #             traders_df=weekly_non_olas_metrics_by_market_creator,
+            #         )
+            #     with gr.Column(scale=1):
+            #         trade_details_text = get_metrics_text(trader_type="non_Olas")
 
-                def update_no_trader_details(trader_detail):
-                    return plot_trader_metrics_by_market_creator(
-                        metric_name=trader_detail,
-                        traders_df=weekly_non_olas_metrics_by_market_creator,
-                    )
+            # def update_no_trader_details(trader_detail):
+            #     return plot_trader_metrics_by_market_creator(
+            #         metric_name=trader_detail,
+            #         traders_df=weekly_non_olas_metrics_by_market_creator,
+            #     )
 
-                trader_no_details_selector.change(
-                    update_no_trader_details,
-                    inputs=trader_no_details_selector,
-                    outputs=trader_no_markets_plot,
-                )
+            # trader_no_details_selector.change(
+            #     update_no_trader_details,
+            #     inputs=trader_no_details_selector,
+            #     outputs=trader_no_markets_plot,
+            # )
             # Unknown traders graph
-            if weekly_unknown_trader_metrics_by_market_creator is not None:
-                with gr.Row():
-                    gr.Markdown("# Weekly metrics of Unclassified traders")
-                with gr.Row():
-                    trader_u_details_selector = gr.Dropdown(
-                        label="Select a weekly trader metric",
-                        choices=trader_metric_choices,
-                        value=default_trader_metric,
-                    )
-
-                with gr.Row():
-                    with gr.Column(scale=3):
-                        trader_u_markets_plot = plot_trader_metrics_by_market_creator(
-                            metric_name=default_trader_metric,
-                            traders_df=weekly_unknown_trader_metrics_by_market_creator,
-                        )
-                    with gr.Column(scale=1):
-                        trade_details_text = get_metrics_text(
-                            trader_type="unclassified"
-                        )
+            # if weekly_unknown_trader_metrics_by_market_creator is not None:
+            #     with gr.Row():
+            #         gr.Markdown("# Weekly metrics of Unclassified traders")
+            #     with gr.Row():
+            #         trader_u_details_selector = gr.Dropdown(
+            #             label="Select a weekly trader metric",
+            #             choices=trader_metric_choices,
+            #             value=default_trader_metric,
+            #         )
 
-                def update_u_trader_details(trader_detail):
-                    return plot_trader_metrics_by_market_creator(
-                        metric_name=trader_detail,
-                        traders_df=weekly_unknown_trader_metrics_by_market_creator,
-                    )
+            #     with gr.Row():
+            #         with gr.Column(scale=3):
+            #             trader_u_markets_plot = plot_trader_metrics_by_agent_categories(
+            #                 metric_name=default_trader_metric,
+            #                 traders_df=weekly_unknown_trader_metrics_by_market_creator,
+            #             )
+            #         with gr.Column(scale=1):
+            #             trade_details_text = get_metrics_text(
+            #                 trader_type="unclassified"
+            #             )
+
+            #     def update_u_trader_details(trader_detail):
+            #         return plot_trader_metrics_by_agent_categories(
+            #             metric_name=trader_detail,
+            #             traders_df=weekly_unknown_trader_metrics_by_market_creator,
+            #         )
 
-                trader_u_details_selector.change(
-                    update_u_trader_details,
-                    inputs=trader_u_details_selector,
-                    outputs=trader_u_markets_plot,
-                )
+            #     trader_u_details_selector.change(
+            #         update_u_trader_details,
+            #         inputs=trader_u_details_selector,
+            #         outputs=trader_u_markets_plot,
+            #     )
 
         with gr.TabItem("📅 Daily metrics"):
             live_trades_current_week = get_current_week_data(trades_df=daily_info)
@@ -664,16 +418,7 @@ with demo:
                         retention_matrix=cohort_retention_olas_qs,
                         cmap="Purples",
                     )
-                    # # non_Olas
-                    # cohort_retention_non_olas_pearl = calculate_cohort_retention(
-                    #     df=retention_df, market_creator="pearl", trader_type="non_Olas"
-                    # )
-                    # if len(cohort_retention_non_olas_pearl) > 0:
-                    #     gr.Markdown("## Cohort retention of Non-Olas traders")
-                    #     cohort_retention_plot2 = plot_cohort_retention_heatmap(
-                    #         retention_matrix=cohort_retention_non_olas_pearl,
-                    #         cmap=sns.color_palette("light:goldenrod", as_cmap=True),
-                    #     )
+
             with gr.Row():
                 with gr.Column(scale=1):
                     gr.Markdown("## Cohort retention of pearl traders")
@@ -701,18 +446,6 @@ with demo:
                             retention_matrix=cohort_retention_unclassified_qs,
                             cmap="Greens",
                         )
-                    # # non_Olas
-                    # cohort_retention_non_olas_qs = calculate_cohort_retention(
-                    #     df=retention_df,
-                    #     market_creator="quickstart",
-                    #     trader_type="non_Olas",
-                    # )
-                    # if len(cohort_retention_non_olas_qs) > 0:
-                    #     gr.Markdown("## Cohort retention of Non-Olas traders")
-                    #     cohort_retention_plot5 = plot_cohort_retention_heatmap(
-                    #         retention_matrix=cohort_retention_non_olas_qs,
-                    #         cmap=sns.color_palette("light:goldenrod", as_cmap=True),
-                    #     )
         with gr.TabItem("⚙️ Active traders"):
             with gr.Row():
                 gr.Markdown("# Active Pearl traders by trader categories")
@@ -750,10 +483,10 @@ with demo:
                 with gr.Column(scale=1):
                     interpretation = get_interpretation_text()
 
-        with gr.TabItem("💰 Money invested per markets"):
+        with gr.TabItem("💰 Money invested per market category"):
             with gr.Row():
                 gr.Markdown("# Weekly total bet amount per trader type for all markets")
-                gr.Markdown("## Computed only for trader agents using the mech service")
+                gr.Markdown("## Computed only for traders using the mech service")
             with gr.Row():
                 total_bet_amount = plot_total_bet_amount(
                     traders_data, market_filter="all"
@@ -777,10 +510,10 @@ with demo:
                     traders_data, market_filter="quickstart"
                 )
 
-        with gr.TabItem("💰 Money invested per market"):
+        with gr.TabItem("💰 Money invested per single market"):
             with gr.Row():
                 gr.Markdown("# Weekly bet amounts per market for all traders")
-                gr.Markdown("## Computed only for trader agents using the mech service")
+                gr.Markdown("## Computed only for traders using the mech service")
             with gr.Row():
                 bet_amounts = plot_total_bet_amount_per_trader_per_market(traders_data)
 
@@ -791,13 +524,6 @@ with demo:
                     traders_data, trader_filter="Olas"
                 )
 
-            # with gr.Row():
-            #     gr.Markdown("# Weekly bet amounts per market for Non-Olas traders")
-            # with gr.Row():
-            #     no_trader_bet_amounts = plot_total_bet_amount_per_trader_per_market(
-            #         traders_data, trader_filter="non_Olas"
-            #     )
-
         with gr.TabItem("🎖️Weekly winning trades % per trader"):
             with gr.Row():
                 gr.Markdown("# Weekly winning trades percentage from all traders")
@@ -815,17 +541,4 @@ with demo:
                     weekly_winning_metrics_olas
                 )
 
-            # # non_Olas traders
-            # if len(weekly_non_olas_winning_metrics) > 0:
-            #     with gr.Row():
-            #         gr.Markdown(
-            #             "# Weekly winning trades percentage from Non-Olas traders"
-            #         )
-            #     with gr.Row():
-            #         metrics_text = get_metrics_text()
-            #     with gr.Row():
-            #         winning_metric = plot_winning_metric_per_trader(
-            #             weekly_non_olas_winning_metrics
-            #         )
-
 demo.queue(default_concurrency_limit=40).launch()

scripts/utils.py

@@ -1,6 +1,10 @@
 import pandas as pd
 from pathlib import Path
 from datetime import datetime, timedelta
+import gzip
+import shutil
+import os
+from huggingface_hub import hf_hub_download
 
 SCRIPTS_DIR = Path(__file__).parent
 ROOT_DIR = SCRIPTS_DIR.parent
@@ -28,3 +32,215 @@ def get_next_week():
     current_date = datetime.now()
     next_week_start = current_date + timedelta(days=7 - (current_date.weekday() + 1))
     return next_week_start.strftime("%b-%d-%Y")
+
+
+def load_all_data():
+
+    # all trades profitability
+    # Download the compressed file
+    gz_filepath_trades = hf_hub_download(
+        repo_id="valory/Olas-predict-dataset",
+        filename="all_trades_profitability.parquet.gz",
+        repo_type="dataset",
+    )
+
+    parquet_filepath_trades = gz_filepath_trades.replace(".gz", "")
+    parquet_filepath_trades = parquet_filepath_trades.replace("all", "")
+
+    with gzip.open(gz_filepath_trades, "rb") as f_in:
+        with open(parquet_filepath_trades, "wb") as f_out:
+            shutil.copyfileobj(f_in, f_out)
+
+    # Now read the decompressed parquet file
+    df1 = pd.read_parquet(parquet_filepath_trades)
+
+    # closed_markets_div
+    closed_markets_df = hf_hub_download(
+        repo_id="valory/Olas-predict-dataset",
+        filename="closed_markets_div.parquet",
+        repo_type="dataset",
+    )
+    df2 = pd.read_parquet(closed_markets_df)
+
+    # daily_info
+    daily_info_df = hf_hub_download(
+        repo_id="valory/Olas-predict-dataset",
+        filename="daily_info.parquet",
+        repo_type="dataset",
+    )
+    df3 = pd.read_parquet(daily_info_df)
+
+    # unknown traders
+    unknown_df = hf_hub_download(
+        repo_id="valory/Olas-predict-dataset",
+        filename="unknown_traders.parquet",
+        repo_type="dataset",
+    )
+    df4 = pd.read_parquet(unknown_df)
+
+    # retention activity
+    gz_file_path_ret = hf_hub_download(
+        repo_id="valory/Olas-predict-dataset",
+        filename="retention_activity.parquet.gz",
+        repo_type="dataset",
+    )
+    parquet_file_path_ret = gz_file_path_ret.replace(".gz", "")
+
+    with gzip.open(gz_file_path_ret, "rb") as f_in:
+        with open(parquet_file_path_ret, "wb") as f_out:
+            shutil.copyfileobj(f_in, f_out)
+    df5 = pd.read_parquet(parquet_file_path_ret)
+    # os.remove(parquet_file_path_ret)
+
+    # active_traders.parquet
+    active_traders_df = hf_hub_download(
+        repo_id="valory/Olas-predict-dataset",
+        filename="active_traders.parquet",
+        repo_type="dataset",
+    )
+    df6 = pd.read_parquet(active_traders_df)
+
+    # weekly_mech_calls.parquet
+    all_mech_calls_df = hf_hub_download(
+        repo_id="valory/Olas-predict-dataset",
+        filename="weekly_mech_calls.parquet",
+        repo_type="dataset",
+    )
+    df7 = pd.read_parquet(all_mech_calls_df)
+
+    # daa for quickstart and pearl
+    daa_qs_df = hf_hub_download(
+        repo_id="valory/Olas-predict-dataset",
+        filename="latest_result_DAA_QS.parquet",
+        repo_type="dataset",
+    )
+    df8 = pd.read_parquet(daa_qs_df)
+
+    daa_pearl_df = hf_hub_download(
+        repo_id="valory/Olas-predict-dataset",
+        filename="latest_result_DAA_Pearl.parquet",
+        repo_type="dataset",
+    )
+    df9 = pd.read_parquet(daa_pearl_df)
+    # Read weekly_avg_roi_pearl_agents.parquet
+    weekly_avg_roi_pearl_agents = hf_hub_download(
+        repo_id="valory/Olas-predict-dataset",
+        filename="weekly_avg_roi_pearl_agents.parquet",
+        repo_type="dataset",
+    )
+    df10 = pd.read_parquet(weekly_avg_roi_pearl_agents)
+
+    # two_weeks_avg_roi_pearl_agents.parquet
+    two_weeks_avg_roi_pearl_agents = hf_hub_download(
+        repo_id="valory/Olas-predict-dataset",
+        filename="two_weeks_avg_roi_pearl_agents.parquet",
+        repo_type="dataset",
+    )
+    df11 = pd.read_parquet(two_weeks_avg_roi_pearl_agents)
+
+    # read traders_weekly_metrics.parquet file
+    traders_weekly_metrics_df = hf_hub_download(
+        repo_id="valory/Olas-predict-dataset",
+        filename="traders_weekly_metrics.parquet",
+        repo_type="dataset",
+    )
+    df12 = pd.read_parquet(traders_weekly_metrics_df)
+    return df1, df2, df3, df4, df5, df6, df7, df8, df9, df10, df11, df12
+
+
+def prepare_data():
+
+    (
+        all_trades,
+        closed_markets,
+        daily_info,
+        unknown_traders,
+        retention_df,
+        active_traders,
+        all_mech_calls,
+        daa_qs_df,
+        daa_pearl_df,
+        weekly_avg_roi_pearl_agents,
+        two_weeks_avg_roi_pearl_agents,
+        traders_weekly_metrics_df,
+    ) = load_all_data()
+    all_trades["creation_timestamp"] = all_trades["creation_timestamp"].dt.tz_convert(
+        "UTC"
+    )
+    all_trades = all_trades.sort_values(by="creation_timestamp", ascending=True)
+    all_trades["creation_date"] = all_trades["creation_timestamp"].dt.date
+
+    # nr-trades variable
+    volume_trades_per_trader_and_market = (
+        all_trades.groupby(["trader_address", "title"])["roi"]
+        .count()
+        .reset_index(name="nr_trades_per_market")
+    )
+
+    traders_data = pd.merge(
+        all_trades, volume_trades_per_trader_and_market, on=["trader_address", "title"]
+    )
+    daily_info["creation_date"] = daily_info["creation_timestamp"].dt.date
+    unknown_traders["creation_date"] = unknown_traders["creation_timestamp"].dt.date
+    active_traders["creation_date"] = active_traders["creation_timestamp"].dt.date
+    # adding the trader family column
+    traders_data["trader_family"] = traders_data.apply(
+        lambda x: get_traders_family(x), axis=1
+    )
+    # print(traders_data.head())
+
+    traders_data = traders_data.sort_values(by="creation_timestamp", ascending=True)
+    unknown_traders = unknown_traders.sort_values(
+        by="creation_timestamp", ascending=True
+    )
+    traders_data["month_year_week"] = (
+        traders_data["creation_timestamp"]
+        .dt.to_period("W")
+        .dt.start_time.dt.strftime("%b-%d-%Y")
+    )
+    unknown_traders["month_year_week"] = (
+        unknown_traders["creation_timestamp"]
+        .dt.to_period("W")
+        .dt.start_time.dt.strftime("%b-%d-%Y")
+    )
+    closed_markets["month_year_week"] = (
+        closed_markets["opening_datetime"]
+        .dt.to_period("W")
+        .dt.start_time.dt.strftime("%b-%d-%Y")
+    )
+
+    # prepare the daa dataframes
+    daa_pearl_df["day"] = pd.to_datetime(
+        daa_pearl_df["day"], format="%Y-%m-%d 00:00:00.000 UTC"
+    )
+    daa_qs_df["day"] = pd.to_datetime(
+        daa_qs_df["day"], format="%Y-%m-%d 00:00:00.000 UTC"
+    )
+    daa_pearl_df["day"] = daa_pearl_df["day"].dt.tz_localize("UTC")
+    daa_qs_df["day"] = daa_qs_df["day"].dt.tz_localize("UTC")
+    daa_qs_df["tx_date"] = pd.to_datetime(daa_qs_df["day"]).dt.date
+    daa_pearl_df["tx_date"] = pd.to_datetime(daa_pearl_df["day"]).dt.date
+    daa_pearl_df["seven_day_trailing_avg"] = pd.to_numeric(
+        daa_pearl_df["seven_day_trailing_avg"], errors="coerce"
+    )
+    daa_pearl_df["seven_day_trailing_avg"] = daa_pearl_df[
+        "seven_day_trailing_avg"
+    ].round(2)
+    daa_qs_df["seven_day_trailing_avg"] = pd.to_numeric(
+        daa_qs_df["seven_day_trailing_avg"], errors="coerce"
+    )
+    daa_qs_df["seven_day_trailing_avg"] = daa_qs_df["seven_day_trailing_avg"].round(2)
+    return (
+        traders_data,
+        closed_markets,
+        daily_info,
+        unknown_traders,
+        retention_df,
+        active_traders,
+        all_mech_calls,
+        daa_qs_df,
+        daa_pearl_df,
+        weekly_avg_roi_pearl_agents,
+        two_weeks_avg_roi_pearl_agents,
+        traders_weekly_metrics_df,
+    )

tabs/trader_plots.py

@@ -84,25 +84,25 @@ def get_interpretation_text() -> gr.Markdown:
     return gr.Markdown(interpretation_text)
 
 
-def plot_trader_metrics_by_market_creator(
+def plot_trader_metrics_by_agent_categories(
     metric_name: str, traders_df: pd.DataFrame
 ) -> gr.Plot:
     """Plots the weekly trader metrics."""
 
     if metric_name == "mech calls":
         metric_name = "mech_calls"
-        column_name = "nr_mech_calls"
+        column_name = "total_mech_calls"
         yaxis_title = "Total nr of mech calls per trader"
     elif metric_name == "ROI":
         column_name = "roi"
         yaxis_title = "Total ROI (net profit/cost)"
     elif metric_name == "bet amount":
         metric_name = "bet_amount"
-        column_name = metric_name
+        column_name = "total_bet_amount"
         yaxis_title = "Total bet amount per trader (xDAI)"
     elif metric_name == "net earnings":
         metric_name = "net_earnings"
-        column_name = metric_name
+        column_name = "net_earnings"
         yaxis_title = "Total net profit per trader (xDAI)"
     elif metric_name == "nr_trades":
         column_name = metric_name
@@ -111,23 +111,10 @@ def plot_trader_metrics_by_market_creator(
         column_name = metric_name
         yaxis_title = "Total gross profit per trader (xDAI)"
 
-    traders_filtered = traders_df[["month_year_week", "market_creator", column_name]]
-    # Convert string dates to datetime and sort them
-    all_dates_dt = sorted(
-        [
-            datetime.strptime(date, "%b-%d-%Y")
-            for date in traders_filtered["month_year_week"].unique()
-        ]
-    )
-    # Convert back to string format
-    all_dates = [date.strftime("%b-%d-%Y") for date in all_dates_dt]
     fig = px.box(
-        traders_filtered,
-        x="month_year_week",
+        traders_df,
+        x="week_start",
         y=column_name,
-        color="market_creator",
-        color_discrete_sequence=["purple", "goldenrod", "darkgreen"],
-        category_orders={"market_creator": ["pearl", "quickstart", "all"]},
     )
     fig.update_traces(boxmean=True)
     fig.update_layout(
@@ -136,8 +123,6 @@ def plot_trader_metrics_by_market_creator(
         legend=dict(yanchor="top", y=0.5),
     )
     fig.update_xaxes(tickformat="%b %d\n%Y")
-    # Update layout to force x-axis category order (hotfix for a sorting issue)
-    fig.update_layout(xaxis={"categoryorder": "array", "categoryarray": all_dates})
 
     return gr.Plot(
         value=fig,