corrections due to week format issues

app.py

@@ -129,28 +129,38 @@ def prepare_data():
     traders_data["trader_family"] = traders_data.apply(
         lambda x: get_traders_family(x), axis=1
     )
-    print(traders_data.head())
+    # print(traders_data.head())
 
     traders_data = traders_data.sort_values(by="creation_timestamp", ascending=True)
     unknown_traders = unknown_traders.sort_values(
         by="creation_timestamp", ascending=True
     )
     traders_data["month_year_week"] = (
-        traders_data["creation_timestamp"].dt.to_period("W").dt.strftime("%b-%d-%Y")
+        traders_data["creation_timestamp"]
+        .dt.to_period("W")
+        .dt.start_time.dt.strftime("%b-%d-%Y")
     )
     unknown_traders["month_year_week"] = (
-        unknown_traders["creation_timestamp"].dt.to_period("W").dt.strftime("%b-%d-%Y")
+        unknown_traders["creation_timestamp"]
+        .dt.to_period("W")
+        .dt.start_time.dt.strftime("%b-%d-%Y")
     )
     closed_markets["month_year_week"] = (
-        closed_markets["opening_datetime"].dt.to_period("W").dt.strftime("%b-%d-%Y")
+        closed_markets["opening_datetime"]
+        .dt.to_period("W")
+        .dt.start_time.dt.strftime("%b-%d-%Y")
     )
     return traders_data, closed_markets, daily_info, unknown_traders, retention_df
 
 
-traders_data, closed_markets, daily_info, unknown_traders, retention_df = prepare_data()
+traders_data, closed_markets, daily_info, unknown_traders, raw_retention_df = (
+    prepare_data()
+)
 retention_df = prepare_retention_dataset(
-    retention_df=retention_df, unknown_df=unknown_traders
+    retention_df=raw_retention_df, unknown_df=unknown_traders
 )
+print("max date of retention df")
+print(max(retention_df.creation_timestamp))
 
 demo = gr.Blocks()
 # get weekly metrics by market creator: qs, pearl or all.
@@ -443,6 +453,10 @@ with demo:
 
             with gr.Row():
                 gr.Markdown("# Cohort retention in pearl traders")
+            with gr.Row():
+                gr.Markdown(
+                    "The Cohort groups are organized by cohort weeks. A trader is part of a cohort group/week where it was detected the FIRST activity ever of that trader."
+                )
             with gr.Row():
                 with gr.Column(scale=1):
                     gr.Markdown("## Cohort retention of 🌊 Olas traders")

data/weekly_mech_calls.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:39b4cf253e9fffc20d584d25410efb8428612413a9f002ba0314216dc56b0ca2
-size 54927
+oid sha256:012e218c54998d9070cc6cbd301ef17188708c37653ec22a6fbff3634d04293d
+size 54922

scripts/num_mech_calls.py

@@ -60,7 +60,9 @@ def compute_total_mech_calls():
         tools["request_date"] = tools["request_time"].dt.date
         tools = tools.sort_values(by="request_time", ascending=True)
         tools["month_year_week"] = (
-            tools["request_time"].dt.to_period("W").dt.strftime("%b-%d-%Y")
+            tools["request_time"]
+            .dt.to_period("W")
+            .dt.start_time.dt.strftime("%b-%d-%Y")
         )
 
     except Exception as e:
@@ -79,7 +81,9 @@ def compute_total_mech_calls():
     fpmmTrades["creation_date"] = fpmmTrades["creation_timestamp"].dt.date
     fpmmTrades = fpmmTrades.sort_values(by="creation_timestamp", ascending=True)
     fpmmTrades["month_year_week"] = (
-        fpmmTrades["creation_timestamp"].dt.to_period("W").dt.strftime("%b-%d-%Y")
+        fpmmTrades["creation_timestamp"]
+        .dt.to_period("W")
+        .dt.start_time.dt.strftime("%b-%d-%Y")
     )
 
     nr_traders = len(fpmmTrades["trader_address"].unique())

scripts/retention_metrics.py

@@ -10,12 +10,21 @@ def calculate_wow_retention_by_type(
     filtered_df = df.loc[df["market_creator"] == market_creator]
     # Get unique traders per week and type
     weekly_traders = (
-        filtered_df.groupby(["month_year_week", "trader_type"])["trader_address"]
+        filtered_df.groupby(["month_year_week", "trader_type"], sort=False)[
+            "trader_address"
+        ]
         .nunique()
         .reset_index()
     )
-    weekly_traders = weekly_traders.sort_values(["trader_type", "month_year_week"])
+    # weekly_traders = weekly_traders.sort_values(['trader_type', 'month_year_week'])
+    # Get ordered list of unique weeks - converting to datetime for proper sorting
+    all_weeks = filtered_df["month_year_week"].unique()
+    weeks_datetime = pd.to_datetime(all_weeks)
+    sorted_weeks_idx = weeks_datetime.argsort()
+    all_weeks = all_weeks[sorted_weeks_idx]
 
+    # Create mapping from week string to numeric index
+    week_to_number = {week: idx for idx, week in enumerate(all_weeks)}
     # Calculate retention
     retention = []
     # Iterate through each trader type
@@ -25,7 +34,20 @@ def calculate_wow_retention_by_type(
         # Calculate retention for each week within this trader type
         for i in range(1, len(type_data)):
             current_week = type_data.iloc[i]["month_year_week"]
-            previous_week = type_data.iloc[i - 1]["month_year_week"]
+            # print(f"current_week={current_week}")
+            week_number = week_to_number[current_week]
+            if week_to_number == 0:
+                # no previous week info
+                continue
+            previous_week_number = week_number - 1
+            # this should give only one value
+            previous_week = [
+                key
+                for key in week_to_number.keys()
+                if week_to_number[key] == previous_week_number
+            ][0]
+            # previous_week = type_data.iloc[i-1]['month_year_week']
+            # print(f"previous week = {previous_week}")
 
             # Get traders in both weeks for this type
             current_traders = set(
@@ -62,7 +84,23 @@ def calculate_wow_retention_by_type(
     return pd.DataFrame(retention)
 
 
-# Cohort Retention
+def create_retention_matrix(cohort_retention_df: pd.DataFrame) -> pd.DataFrame:
+    # Pivot the data to create the retention matrix
+    retention_matrix = cohort_retention_df.pivot(
+        index="cohort_week", columns="weeks_since_cohort", values="retention_rate"
+    )
+
+    # Sort index chronologically
+    retention_matrix.index = pd.to_datetime(retention_matrix.index)
+    retention_matrix = retention_matrix.sort_index()
+
+    # Rename columns to show week numbers
+    # retention_matrix.columns = [f"Week {i}" for i in retention_matrix.columns]
+
+    return retention_matrix
+
+
+# Wow Retention at the cohort level
 def calculate_cohort_retention(
     df: pd.DataFrame, market_creator: str, trader_type: str
 ) -> pd.DataFrame:
@@ -70,7 +108,7 @@ def calculate_cohort_retention(
         (df["market_creator"] == market_creator) & (df["trader_type"] == trader_type)
     ]
     df_filtered = df_filtered.sort_values(by="creation_timestamp", ascending=True)
-    # Get first week for each trader
+    # Get first week of activity for each trader
     first_activity = (
         df_filtered.groupby("trader_address")
         .agg({"creation_timestamp": "min", "month_year_week": "first"})
@@ -94,54 +132,82 @@ def calculate_cohort_retention(
         on="trader_address",
     )
 
-    # Calculate week number since first activity
-    cohort_data["cohort_number"] = cohort_data["cohort_week"].map(week_to_number)
-    cohort_data["activity_number"] = cohort_data["month_year_week"].map(week_to_number)
-    cohort_data["week_number"] = (
-        cohort_data["activity_number"] - cohort_data["cohort_number"]
-    )
-
-    # Calculate retention by cohort
-    cohort_sizes = cohort_data.groupby("cohort_week")["trader_address"].nunique()
-    retention_matrix = cohort_data.groupby(["cohort_week", "week_number"])[
-        "trader_address"
-    ].nunique()
-    retention_matrix = retention_matrix.unstack(fill_value=0)
+    # Get all unique weeks and cohorts
+    all_cohorts = cohort_data["cohort_week"].unique()
+    # print(f"all cohorts = {all_cohorts}")
+    retention_data = []
+    max_weeks = 8
+    for cohort in all_cohorts:
+        # print(f"analyzing cohort {cohort}")
+        # Get all traders in this cohort
+        cohort_traders = set(
+            cohort_data[cohort_data["cohort_week"] == cohort]["trader_address"]
+        )
+        cohort_size = len(cohort_traders)
+        # print(f"cohort size = {cohort_size}")
 
-    # Convert to percentages
-    retention_matrix = retention_matrix.div(cohort_sizes, axis=0) * 100
+        if cohort_size == 0:
+            print(f"NO new traders for cohort week={cohort}")
+            continue
 
-    # Sort index (cohort_week) chronologically
-    retention_matrix.index = pd.to_datetime(retention_matrix.index)
-    retention_matrix = retention_matrix.sort_index()
-
-    return retention_matrix.round(2)
+        # Calculate retention for each week after the cohort week
+        for week_idx, week in enumerate(all_weeks):
+            # print(f"Analyzing week = {week}")
+            weeks_since_cohort = week_idx - week_to_number[cohort]
+            if weeks_since_cohort < 0 or weeks_since_cohort > 8:
+                continue
+            # Get active traders from the cohort in current week
+            current_traders = set(
+                cohort_data[
+                    (cohort_data["cohort_week"] == cohort)
+                    & (cohort_data["month_year_week"] == week)
+                ]["trader_address"]
+            )
 
+            # Get active traders from the cohort in previous week
+            if week == cohort:
+                # For the first week, retention is 100% by definition
+                retained = len(current_traders)
+                retention_rate = 100 if len(current_traders) > 0 else 0
 
-def merge_retention_dataset(
-    traders_df: pd.DataFrame, unknown_df: pd.DataFrame
-) -> pd.DataFrame:
+            elif week_idx > 0:
+                previous_week = all_weeks[week_idx - 1]
+                previous_traders = set(
+                    cohort_data[
+                        (cohort_data["cohort_week"] == cohort)
+                        & (cohort_data["month_year_week"] == previous_week)
+                    ]["trader_address"]
+                )
+                retained = len(current_traders.intersection(previous_traders))
+                retention_rate = (
+                    (retained / len(previous_traders)) * 100
+                    if len(previous_traders) > 0
+                    else 0
+                )
 
-    traders_df["trader_type"] = traders_df["staking"].apply(
-        lambda x: "non_Olas" if x == "non_Olas" else "Olas"
-    )
-    unknown_df["trader_type"] = "unclassified"
-    all_traders = pd.concat([traders_df, unknown_df], ignore_index=True)
+            retention_data.append(
+                {
+                    "cohort_week": cohort,
+                    "week": week,
+                    "weeks_since_cohort": weeks_since_cohort,
+                    "cohort_size": cohort_size,
+                    "active_traders": len(current_traders),
+                    "retained_traders": retained,
+                    "previous_traders": (
+                        len(previous_traders) if week_idx > 0 else cohort_size
+                    ),
+                    "retention_rate": round(retention_rate, 2),
+                }
+            )
 
-    all_traders["creation_timestamp"] = pd.to_datetime(
-        all_traders["creation_timestamp"]
-    )
-    all_traders = all_traders.sort_values(by="creation_timestamp", ascending=True)
-    all_traders["month_year_week"] = (
-        all_traders["creation_timestamp"].dt.to_period("W").dt.strftime("%b-%d-%Y")
-    )
-    return all_traders
+    retention_matrix = create_retention_matrix(pd.DataFrame(retention_data))
+    return retention_matrix
 
 
 def prepare_retention_dataset(
     retention_df: pd.DataFrame, unknown_df: pd.DataFrame
 ) -> pd.DataFrame:
-
+    print("Preparing retention dataset")
     retention_df["trader_type"] = retention_df["staking"].apply(
         lambda x: "non_Olas" if x == "non_Olas" else "Olas"
     )
@@ -158,11 +224,28 @@ def prepare_retention_dataset(
     all_traders["creation_timestamp"] = pd.to_datetime(
         all_traders["creation_timestamp"]
     )
+
     all_traders = all_traders.sort_values(by="creation_timestamp", ascending=True)
-    all_traders["month_year_week"] = (
-        all_traders["creation_timestamp"].dt.to_period("W").dt.strftime("%b-%d-%Y")
+
+    # Remove data from current week and onwards
+    now = datetime.now()
+
+    # Get start of the current week (Monday)
+    start_of_week = now - timedelta(days=(now.weekday()))
+    start_of_week = start_of_week.replace(hour=0, minute=0, second=0, microsecond=0)
+
+    all_traders["creation_date"] = all_traders["creation_timestamp"].dt.date
+    all_traders["creation_date"] = pd.to_datetime(all_traders["creation_date"])
+    # Filter the dataframe
+    filtered_traders = all_traders[all_traders["creation_date"] < start_of_week]
+    filtered_traders["month_year_week"] = (
+        filtered_traders["creation_timestamp"]
+        .dt.to_period("W")
+        .dt.start_time.dt.strftime("%b-%d-%Y")
     )
-    return all_traders
+
+    print(filtered_traders.month_year_week.unique())
+    return filtered_traders
 
 
 if __name__ == "__main__":

tabs/daily_graphs.py

@@ -19,15 +19,6 @@ color_mapping = [
 ]
 
 
-def plot_daily_trades(trades_df: pd.DataFrame) -> gr.Plot:
-    # get daily trades
-    daily_trades_count = (
-        trades_df.groupby("month_year_week").size().reset_index(name="trades")
-    )
-    daily_trades_count.columns = daily_trades_count.columns.astype(str)
-    print("WIP")
-
-
 def get_current_week_data(trades_df: pd.DataFrame) -> pd.DataFrame:
     # Get current date
     now = datetime.now()
@@ -35,12 +26,12 @@ def get_current_week_data(trades_df: pd.DataFrame) -> pd.DataFrame:
     # Get start of the current week (Monday)
     start_of_week = now - timedelta(days=now.weekday())
     start_of_week = start_of_week.replace(hour=0, minute=0, second=0, microsecond=0)
-    print(f"start of the week = {start_of_week}")
+    # print(f"start of the week = {start_of_week}")
 
     # Get end of the current week (Sunday)
     end_of_week = start_of_week + timedelta(days=6)
     end_of_week = end_of_week.replace(hour=23, minute=59, second=59, microsecond=999999)
-    print(f"end of the week = {end_of_week}")
+    # print(f"end of the week = {end_of_week}")
     trades_df["creation_date"] = pd.to_datetime(trades_df["creation_date"])
     # Filter the dataframe
     return trades_df[