use duckdb

app.py

@@ -1,6 +1,7 @@
-from datetime import datetime
+from datetime import datetime, timedelta
 import gradio as gr
 import pandas as pd
+import duckdb
 from tabs.trades import (
     prepare_trades, 
     get_overall_trades, 
@@ -24,20 +25,35 @@ from tabs.error import (
 )
 from tabs.about import about_olas_predict
 
+
+def get_last_two_months_data():
+    con = duckdb.connect(':memory:')
+    two_months_ago = (datetime.now() - timedelta(days=60)).strftime('%Y-%m-%d')
+    
+    query1 = f"""
+    SELECT *
+    FROM read_parquet('./data/tools.parquet')
+    WHERE request_time >= '{two_months_ago}'
+    """
+    query2 = f"""
+    SELECT *
+    FROM read_parquet('./data/all_trades_profitability.parquet')
+    WHERE creation_timestamp >= '{two_months_ago}'
+    """
+
+    df1 = con.execute(query1).fetchdf()
+    df2 = con.execute(query2).fetchdf()
+    con.close()
+    return df1, df2
         
 def prepare_data():
-    tools_df = pd.read_parquet("./data/tools.parquet")
-    trades_df = pd.read_parquet("./data/all_trades_profitability.parquet")
+    tools_df, trades_df = get_last_two_months_data()
 
     # get current month
     current_month = datetime.now().strftime("%Y-%m")
 
     tools_df['request_time'] = pd.to_datetime(tools_df['request_time'])
-    # only keep trades for the current month
-    tools_df = tools_df[tools_df['request_time'].dt.strftime('%Y-%m') == current_month]
-
     trades_df['creation_timestamp'] = pd.to_datetime(trades_df['creation_timestamp'])
-    trades_df = trades_df[trades_df['creation_timestamp'].dt.strftime('%Y-%m') == current_month]
 
     trades_df = prepare_trades(trades_df)
     return tools_df, trades_df

requirements.txt

@@ -5,3 +5,4 @@ pyarrow
 requests
 gradio==4.13.0
 pytz
+duckdb