trying to fix error with loading files

app.py

@@ -65,21 +65,21 @@ def load_all_data():
 
     # all trades profitability
     # Download the compressed file
-    gz_file_path = hf_hub_download(
+    gz_file_path_trades = hf_hub_download(
         repo_id="valory/Olas-predict-dataset",
         filename="all_trades_profitability.parquet.gz",
         repo_type="dataset",
     )
 
-    parquet_file_path = gz_file_path.replace(".gz", "")
+    parquet_file_path_trades = gz_file_path_trades.replace(".gz", "")
+    parquet_file_path_trades = parquet_file_path_trades.replace("all", "")
 
-    with gzip.open(gz_file_path, "rb") as f_in:
-        with open(parquet_file_path, "wb") as f_out:
+    with gzip.open(gz_file_path_trades, "rb") as f_in:
+        with open(parquet_file_path_trades, "wb") as f_out:
             shutil.copyfileobj(f_in, f_out)
 
     # Now read the decompressed parquet file
-    df1 = pd.read_parquet(parquet_file_path)
-    os.remove(parquet_file_path)
+    df1 = pd.read_parquet(parquet_file_path_trades)
 
     # closed_markets_div
     closed_markets_df = hf_hub_download(
@@ -106,18 +106,18 @@ def load_all_data():
     df4 = pd.read_parquet(unknown_df)
 
     # retention activity
-    gz_file_path2 = hf_hub_download(
+    gz_file_path_ret = hf_hub_download(
         repo_id="valory/Olas-predict-dataset",
         filename="retention_activity.parquet.gz",
         repo_type="dataset",
     )
-    parquet_file_path2 = gz_file_path2.replace(".gz", "")
+    parquet_file_path_ret = gz_file_path_ret.replace(".gz", "")
 
-    with gzip.open(gz_file_path2, "rb") as f_in:
-        with open(parquet_file_path2, "wb") as f_out:
+    with gzip.open(gz_file_path_ret, "rb") as f_in:
+        with open(parquet_file_path_ret, "wb") as f_out:
             shutil.copyfileobj(f_in, f_out)
-    df5 = pd.read_parquet(parquet_file_path2)
-    os.remove(parquet_file_path2)
+    df5 = pd.read_parquet(parquet_file_path_ret)
+    # os.remove(parquet_file_path_ret)
 
     # active_traders.parquet
     active_traders_df = hf_hub_download(
@@ -208,6 +208,7 @@ def prepare_data():
     active_traders,
     all_mech_calls,
 ) = prepare_data()
+
 retention_df = prepare_retention_dataset(
     retention_df=raw_retention_df, unknown_df=unknown_traders
 )

requirements.txt

@@ -10,4 +10,5 @@ pydantic
 pydantic_core
 nbformat
 pytz
+duckdb
 ipfshttpclient

scripts/retention_metrics.py

@@ -1,6 +1,5 @@
 import pandas as pd
 from datetime import datetime, timedelta
-from scripts.utils import DATA_DIR
 
 
 # Basic Week over Week Retention
@@ -263,10 +262,4 @@ def prepare_retention_dataset(
 
 
 if __name__ == "__main__":
-    # read all datasets
-    traders_df = pd.read_parquet(DATA_DIR / "all_trades_profitability.parquet")
-    unknown_df = pd.read_parquet(DATA_DIR / "unknown_traders.parquet")
-    all_traders = prepare_retention_dataset(traders_df, unknown_df)
-    # Usage example:
-    wow_retention = calculate_wow_retention_by_type(all_traders)
-    cohort_retention = calculate_cohort_retention(all_traders)
+    print("WIP")

scripts/utils.py

@@ -4,7 +4,6 @@ from datetime import datetime, timedelta
 
 SCRIPTS_DIR = Path(__file__).parent
 ROOT_DIR = SCRIPTS_DIR.parent
-DATA_DIR = ROOT_DIR / "data"
 TMP_DIR = ROOT_DIR / "tmp"