new tools accuracy information

app.py

@@ -26,6 +26,8 @@ from tabs.error import (
 )
 from tabs.about import about_olas_predict, about_this_dashboard
 
+from scripts.utils import INC_TOOLS
+
 
 def get_logger():
     logger = logging.getLogger(__name__)
@@ -119,22 +121,6 @@ tools_df, trades_df = prepare_data()
 
 demo = gr.Blocks()
 
-
-INC_TOOLS = [
-    "prediction-online",
-    "prediction-offline",
-    "claude-prediction-online",
-    "claude-prediction-offline",
-    "prediction-offline-sme",
-    "prediction-online-sme",
-    "prediction-request-rag",
-    "prediction-request-reasoning",
-    "prediction-url-cot-claude",
-    "prediction-request-rag-claude",
-    "prediction-request-reasoning-claude",
-]
-
-
 error_df = get_error_data(tools_df=tools_df, inc_tools=INC_TOOLS)
 error_overall_df = get_error_data_overall(error_df=error_df)
 winning_rate_df = get_tool_winning_rate(tools_df=tools_df, inc_tools=INC_TOOLS)

data/all_trades_profitability.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:906e77fed8d2af6d7a4f4acf73640ce6aa3e3d4357e01c7b00af1c180f3b6eaf
-size 2460157
+oid sha256:78ca1c4b1ad00c95f94bdaecef627d6449547984d18e452f2db65fa194417a52
+size 2456352

data/delivers.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2d975d5fc2c3c7c50bec7136dffa135aef50fe3802bd1650cd46f236675318af
-size 510466826
+oid sha256:cfad460f3b035878c083a8a98988b0c623643ddf688b364d6269f45c194beae8
+size 561642110

data/fpmmTrades.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:781a9625add04d8337bb3909befb6c0d4c39f3e0c9a5c47c1fb9398a96ba36ae
-size 6482896
+oid sha256:295b4f6839111251a764a5fcde484552daaa6b42c63e5753680a33311c53b4e5
+size 6484981

data/fpmms.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a2c07ed22be9463c8465d34717222f1553c5890254b350fb07aec2604e925795
-size 336922
+oid sha256:c488af7d0ad3070526751daafce3de6c944f9be289503f885ec6a4d4336183b2
+size 338750

data/requests.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bff6dbad037f5f9c3cee3631c126ba0f6e582b32b7e3fb75d79fc60a43a9dadb
-size 11547978
+oid sha256:f6c5aa48864ff7eb8662d3ec1faff6e533d4067eadab72a16be233adf5df0216
+size 16857932

data/summary_profitability.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c5db58ef61b7a256437ee87238249cd1da9d229a4ca5f9af79b02efe61fa73cc
-size 40127
+oid sha256:b82f7c4a487ae6b7d8daadda5454f4f8327b83ca85bd9d252d32a8d39105a526
+size 42145

data/t_map.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:29c60bf39a93cad87fd54b7a87407cb8f50bb2cd2122e993762cec36d76e76fb
-size 8982599
+oid sha256:2dc45f8d93c69a51da90edd37aacff70ef5b9cb2680201d0e76711e245d2b240
+size 9252439

data/tools.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cf3ed51b3e32b32e3cd9bba607afbc1e6d67583e6ca9126df68585c339964a8a
-size 378117330
+oid sha256:bc017439308d15c93dda7c34cfb46b5ff4de1cbe3614dc2ff18fab7b50622e14
+size 562391192

data/tools_accuracy.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18c334d2f2db204a97501adde354c61569b269514579781f894787e52829407d
+size 1014

notebooks/analysis.ipynb

@@ -475,6 +475,20 @@
     "total"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "total.to_csv(\"accuracy_info.csv\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": []
+  },
   {
    "cell_type": "code",
    "execution_count": 3,

requirements.txt

@@ -5,4 +5,5 @@ pyarrow
 requests
 gradio==4.13.0
 pytz
-duckdb
+duckdb
+ipfshttpclient

scripts/pull_data.py

@@ -19,9 +19,9 @@ from tools import (
     DEFAULT_FILENAME as TOOLS_FILENAME,
     update_tools_accuracy,
 )
-from app import INC_TOOLS
+from utils import INC_TOOLS
 from profitability import run_profitability_analysis
-
+import ipfshttpclient
 import gc
 
 logging.basicConfig(level=logging.INFO)
@@ -153,15 +153,6 @@ def weekly_analysis():
     with open(DATA_DIR / "t_map.pkl", "wb") as f:
         pickle.dump(t_map, f)
 
-    # Computing tools accuracy information
-    print("Computing tool accuracy information")
-    # Check if the file exists
-    acc_data = None
-    if os.path.exists(DATA_DIR / ACCURACY_FILENAME):
-        acc_data = pd.read_csv(DATA_DIR / ACCURACY_FILENAME)
-    update_tools_accuracy(acc_data, tools, INC_TOOLS)
-    # TODO save acc_data into a CSV file
-
     # clean and release all memory
     del tools
     del fpmms

scripts/tools.py

@@ -501,20 +501,43 @@ def update_tools_accuracy(
         tools_non_error["currentAnswer"] == tools_non_error["vote"]
     ).astype(int)
     tools_non_error.columns = tools_non_error.columns.astype(str)
+    print("Tools dataset after filtering")
+    print(tools_non_error.head())
+
     wins = tools_non_error.groupby(["tool", "win"]).size().unstack().fillna(0)
     wins["tool_accuracy"] = (wins[1] / (wins[0] + wins[1])) * 100
     wins.reset_index(inplace=True)
     wins["total_requests"] = wins[0] + wins[1]
     wins.columns = wins.columns.astype(str)
     wins = wins[["tool", "tool_accuracy", "total_requests"]]
+
+    print("Wins dataset")
+    print(wins.head())
     timeline = tools_non_error.groupby(["tool"])["request_time"].agg(["min", "max"])
+
+    print("timeline dataset")
+    print(timeline.head())
     acc_info = wins.merge(timeline, how="left", on="tool")
 
     if tools_acc is None:
         print("Creating accuracy file for the first time")
         return acc_info
 
-    # TODO update the old information
+    # update the old information
+    print("Updating accuracy information")
+    tools_to_update = list(acc_info["tool"].values)
+    existing_tools = list(tools_acc["tool"].values)
+    for tool in tools_to_update:
+        if tool in existing_tools:
+            new_accuracy = acc_info[acc_info["tool"] == tool, "tool_accuracy"]
+            new_volume = acc_info[acc_info["tool"] == tool, "total_requests"]
+            new_min_timeline = acc_info[acc_info["tool"] == tool, "min"]
+            new_max_timeline = acc_info[acc_info["tool"] == tool, "max"]
+            tools_acc[tools_acc["tool"] == tool, "tool_accuracy"] = new_accuracy
+            tools_acc[tools_acc["tool"] == tool, "total_requests"] = new_volume
+            tools_acc[tools_acc["tool"] == tool, "min"] = new_min_timeline
+            tools_acc[tools_acc["tool"] == tool, "max"] = new_max_timeline
+    return tools_acc
 
 
 if __name__ == "__main__":

scripts/update_tools_accuracy.py

@@ -0,0 +1,34 @@
+import os
+import pandas as pd
+import ipfshttpclient
+from pull_data import DATA_DIR
+from utils import INC_TOOLS
+from tools import update_tools_accuracy
+
+ACCURACY_FILENAME = "tools_accuracy.csv"
+IPFS_SERVER = "/dns/registry.autonolas.tech/tcp/443/https"
+
+
+def compute_tools_accuracy():
+    print("Reading tools parquet file")
+    tools = pd.read_parquet(DATA_DIR / "tools.parquet")
+    print(tools.head())
+    # Computing tools accuracy information
+    print("Computing tool accuracy information")
+    # Check if the file exists
+    acc_data = None
+    if os.path.exists(DATA_DIR / ACCURACY_FILENAME):
+        acc_data = pd.read_csv(DATA_DIR / ACCURACY_FILENAME)
+    acc_data = update_tools_accuracy(acc_data, tools, INC_TOOLS)
+    # save acc_data into a CSV file
+    print("Saving into a csv file")
+    acc_data.to_csv(DATA_DIR / ACCURACY_FILENAME, index=False)
+
+    # save the data into IPFS
+    client = ipfshttpclient.connect(IPFS_SERVER)
+    result = client.add(DATA_DIR / ACCURACY_FILENAME)
+    print(f"HASH of the tools accuracy file: {result['Hash']}")
+
+
+if __name__ == "__main__":
+    compute_tools_accuracy()

scripts/utils.py

@@ -27,6 +27,20 @@ HTTPS = HTTP[:4] + "s" + HTTP[4:]
 IPFS_ADDRESS = f"{HTTPS}gateway.autonolas.tech/ipfs/"
 FORMAT_UPDATE_BLOCK_NUMBER = 30411638
 
+INC_TOOLS = [
+    "prediction-online",
+    "prediction-offline",
+    "claude-prediction-online",
+    "claude-prediction-offline",
+    "prediction-offline-sme",
+    "prediction-online-sme",
+    "prediction-request-rag",
+    "prediction-request-reasoning",
+    "prediction-url-cot-claude",
+    "prediction-request-rag-claude",
+    "prediction-request-reasoning-claude",
+]
+
 
 class MechEventName(Enum):
     """The mech's event names."""