cleaning and refactoring code

scripts/pull_data.py

@@ -13,9 +13,7 @@ from markets import (
     DEFAULT_FILENAME as MARKETS_FILENAME,
 )
 from tools import (
-    etl as tools_etl,
     DEFAULT_FILENAME as TOOLS_FILENAME,
-    update_tools_accuracy,
     generate_tools_file,
 )
 from profitability import run_profitability_analysis

scripts/roi_analysis.py

@@ -13,7 +13,6 @@ from markets import (
 TOOLS_FILENAME = "tools_2024.parquet"
 from tools import (
     etl as tools_etl,
-    update_tools_accuracy,
 )
 from pull_data import (
     DATA_DIR,

scripts/tools.py

@@ -613,77 +613,6 @@ def generate_tools_file():
         print(f"An Exception happened while parsing the json events {e}")
 
 
-def update_tools_accuracy(
-    tools_acc: pd.DataFrame, tools_df: pd.DataFrame, inc_tools: List[str]
-) -> pd.DataFrame:
-    """To compute/update the latest accuracy information for the different mech tools"""
-
-    # computation of the accuracy information
-    tools_inc = tools_df[tools_df["tool"].isin(inc_tools)]
-    # filtering errors
-    tools_non_error = tools_inc[tools_inc["error"] != 1]
-    tools_non_error.loc[:, "currentAnswer"] = tools_non_error["currentAnswer"].replace(
-        {"no": "No", "yes": "Yes"}
-    )
-    tools_non_error = tools_non_error[
-        tools_non_error["currentAnswer"].isin(["Yes", "No"])
-    ]
-    tools_non_error = tools_non_error[tools_non_error["vote"].isin(["Yes", "No"])]
-    tools_non_error["win"] = (
-        tools_non_error["currentAnswer"] == tools_non_error["vote"]
-    ).astype(int)
-    tools_non_error.columns = tools_non_error.columns.astype(str)
-    print("Tools dataset after filtering")
-    print(tools_non_error.head())
-
-    wins = tools_non_error.groupby(["tool", "win"]).size().unstack().fillna(0)
-    wins["tool_accuracy"] = (wins[1] / (wins[0] + wins[1])) * 100
-    wins.reset_index(inplace=True)
-    wins["total_requests"] = wins[0] + wins[1]
-    wins.columns = wins.columns.astype(str)
-    wins = wins[["tool", "tool_accuracy", "total_requests"]]
-
-    print("Wins dataset")
-    print(wins.head())
-    no_timeline_info = False
-    try:
-        timeline = tools_non_error.groupby(["tool"])["request_time"].agg(["min", "max"])
-        print("timeline dataset")
-        print(timeline.head())
-        acc_info = wins.merge(timeline, how="left", on="tool")
-    except:
-        print("NO REQUEST TIME INFORMATION AVAILABLE")
-        no_timeline_info = True
-        acc_info = wins
-
-    if tools_acc is None:
-        print("Creating accuracy file for the first time")
-        return acc_info
-
-    # update the old information
-    print("Updating accuracy information")
-    tools_to_update = list(acc_info["tool"].values)
-    print("tools to update")
-    print(tools_to_update)
-    existing_tools = list(tools_acc["tool"].values)
-    for tool in tools_to_update:
-        if tool in existing_tools:
-            new_accuracy = acc_info[acc_info["tool"] == tool]["tool_accuracy"].values[0]
-            new_volume = acc_info[acc_info["tool"] == tool]["total_requests"].values[0]
-            if no_timeline_info:
-                new_min_timeline = None
-                new_max_timeline = None
-            else:
-                new_min_timeline = acc_info[acc_info["tool"] == tool]["min"].values[0]
-                new_max_timeline = acc_info[acc_info["tool"] == tool]["max"].values[0]
-            tools_acc.loc[tools_acc["tool"] == tool, "tool_accuracy"] = new_accuracy
-            tools_acc.loc[tools_acc["tool"] == tool, "total_requests"] = new_volume
-            tools_acc.loc[tools_acc["tool"] == tool, "min"] = new_min_timeline
-            tools_acc.loc[tools_acc["tool"] == tool, "max"] = new_max_timeline
-    print(tools_acc)
-    return tools_acc
-
-
 if __name__ == "__main__":
     RPCs = [
         "https://lb.nodies.app/v1/406d8dcc043f4cb3959ed7d6673d311a",

scripts/update_tools_accuracy.py

@@ -3,7 +3,7 @@ import pandas as pd
 import ipfshttpclient
 from pathlib import Path
 from utils import INC_TOOLS
-from tools import update_tools_accuracy
+from typing import List
 
 ACCURACY_FILENAME = "tools_accuracy.csv"
 IPFS_SERVER = "/dns/registry.autonolas.tech/tcp/443/https"
@@ -12,6 +12,77 @@ ROOT_DIR = SCRIPTS_DIR.parent
 DATA_DIR = ROOT_DIR / "data"
 
 
+def update_tools_accuracy(
+    tools_acc: pd.DataFrame, tools_df: pd.DataFrame, inc_tools: List[str]
+) -> pd.DataFrame:
+    """To compute/update the latest accuracy information for the different mech tools"""
+
+    # computation of the accuracy information
+    tools_inc = tools_df[tools_df["tool"].isin(inc_tools)]
+    # filtering errors
+    tools_non_error = tools_inc[tools_inc["error"] != 1]
+    tools_non_error.loc[:, "currentAnswer"] = tools_non_error["currentAnswer"].replace(
+        {"no": "No", "yes": "Yes"}
+    )
+    tools_non_error = tools_non_error[
+        tools_non_error["currentAnswer"].isin(["Yes", "No"])
+    ]
+    tools_non_error = tools_non_error[tools_non_error["vote"].isin(["Yes", "No"])]
+    tools_non_error["win"] = (
+        tools_non_error["currentAnswer"] == tools_non_error["vote"]
+    ).astype(int)
+    tools_non_error.columns = tools_non_error.columns.astype(str)
+    print("Tools dataset after filtering")
+    print(tools_non_error.head())
+
+    wins = tools_non_error.groupby(["tool", "win"]).size().unstack().fillna(0)
+    wins["tool_accuracy"] = (wins[1] / (wins[0] + wins[1])) * 100
+    wins.reset_index(inplace=True)
+    wins["total_requests"] = wins[0] + wins[1]
+    wins.columns = wins.columns.astype(str)
+    wins = wins[["tool", "tool_accuracy", "total_requests"]]
+
+    print("Wins dataset")
+    print(wins.head())
+    no_timeline_info = False
+    try:
+        timeline = tools_non_error.groupby(["tool"])["request_time"].agg(["min", "max"])
+        print("timeline dataset")
+        print(timeline.head())
+        acc_info = wins.merge(timeline, how="left", on="tool")
+    except:
+        print("NO REQUEST TIME INFORMATION AVAILABLE")
+        no_timeline_info = True
+        acc_info = wins
+
+    if tools_acc is None:
+        print("Creating accuracy file for the first time")
+        return acc_info
+
+    # update the old information
+    print("Updating accuracy information")
+    tools_to_update = list(acc_info["tool"].values)
+    print("tools to update")
+    print(tools_to_update)
+    existing_tools = list(tools_acc["tool"].values)
+    for tool in tools_to_update:
+        if tool in existing_tools:
+            new_accuracy = acc_info[acc_info["tool"] == tool]["tool_accuracy"].values[0]
+            new_volume = acc_info[acc_info["tool"] == tool]["total_requests"].values[0]
+            if no_timeline_info:
+                new_min_timeline = None
+                new_max_timeline = None
+            else:
+                new_min_timeline = acc_info[acc_info["tool"] == tool]["min"].values[0]
+                new_max_timeline = acc_info[acc_info["tool"] == tool]["max"].values[0]
+            tools_acc.loc[tools_acc["tool"] == tool, "tool_accuracy"] = new_accuracy
+            tools_acc.loc[tools_acc["tool"] == tool, "total_requests"] = new_volume
+            tools_acc.loc[tools_acc["tool"] == tool, "min"] = new_min_timeline
+            tools_acc.loc[tools_acc["tool"] == tool, "max"] = new_max_timeline
+    print(tools_acc)
+    return tools_acc
+
+
 def compute_tools_accuracy():
     print("Computing accuracy of tools")
     print("Reading tools parquet file")