updating dialy data

data/closed_markets_div.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8df376959d26e13d47dcbe3c25acc82f028e35f33ef7928a869eed102d8c7e34
-size 64570
+oid sha256:1358e72ea2c93185bb28f6660d238fd745d0d9a7a74bdd52daf14eca6bd65140
+size 64772

data/daily_info.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:66f689a59cd158b095e78225ae6d85977771c2a7f9ad323a6692e33e34d16b05
-size 835096
+oid sha256:195d5d8c5252f2c663eb5226e70074515124ac3dce174fbc153eb7dd85b2ba00
+size 499139

notebooks/closed_markets.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -11,24 +11,32 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 3,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Error reading the parquet file\n"
+     ]
+    }
+   ],
    "source": [
     "try:\n",
     "    markets = pd.read_parquet(\"../data/fpmms.parquet\")\n",
     "except Exception:\n",
     "    print(\"Error reading the parquet file\")\n",
     "\n",
-    "markets[\"currentAnswer\"] = markets[\"currentAnswer\"].apply(lambda x: x.lower())\n",
-    "# filter only markets with  yes, no answers\n",
-    "valid_answers = [\"yes\", \"no\"]\n",
-    "markets = markets.loc[markets[\"currentAnswer\"].isin(valid_answers)]"
+    "# markets[\"currentAnswer\"] = markets[\"currentAnswer\"].apply(lambda x: x.lower())\n",
+    "# # filter only markets with  yes, no answers\n",
+    "# valid_answers = [\"yes\", \"no\"]\n",
+    "# markets = markets.loc[markets[\"currentAnswer\"].isin(valid_answers)]"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -40,7 +48,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
@@ -48,36 +56,36 @@
      "output_type": "stream",
      "text": [
       "<class 'pandas.core.frame.DataFrame'>\n",
-      "RangeIndex: 61442 entries, 0 to 61441\n",
+      "RangeIndex: 85876 entries, 0 to 85875\n",
       "Data columns (total 24 columns):\n",
       " #   Column                         Non-Null Count  Dtype \n",
       "---  ------                         --------------  ----- \n",
-      " 0   collateralAmount               61442 non-null  object\n",
-      " 1   collateralAmountUSD            61442 non-null  object\n",
-      " 2   collateralToken                61442 non-null  object\n",
-      " 3   creationTimestamp              61442 non-null  object\n",
-      " 4   trader_address                 61442 non-null  object\n",
-      " 5   feeAmount                      61442 non-null  object\n",
-      " 6   id                             61442 non-null  object\n",
-      " 7   oldOutcomeTokenMarginalPrice   61442 non-null  object\n",
-      " 8   outcomeIndex                   61442 non-null  object\n",
-      " 9   outcomeTokenMarginalPrice      61442 non-null  object\n",
-      " 10  outcomeTokensTraded            61442 non-null  object\n",
-      " 11  title                          61442 non-null  object\n",
-      " 12  transactionHash                61442 non-null  object\n",
-      " 13  type                           61442 non-null  object\n",
-      " 14  market_creator                 61442 non-null  object\n",
-      " 15  fpmm.answerFinalizedTimestamp  47113 non-null  object\n",
-      " 16  fpmm.arbitrationOccurred       61442 non-null  bool  \n",
-      " 17  fpmm.currentAnswer             47113 non-null  object\n",
-      " 18  fpmm.id                        61442 non-null  object\n",
-      " 19  fpmm.isPendingArbitration      61442 non-null  bool  \n",
-      " 20  fpmm.openingTimestamp          61442 non-null  object\n",
-      " 21  fpmm.outcomes                  61442 non-null  object\n",
-      " 22  fpmm.title                     61442 non-null  object\n",
-      " 23  fpmm.condition.id              61442 non-null  object\n",
+      " 0   collateralAmount               85876 non-null  object\n",
+      " 1   collateralAmountUSD            85876 non-null  object\n",
+      " 2   collateralToken                85876 non-null  object\n",
+      " 3   creationTimestamp              85876 non-null  object\n",
+      " 4   trader_address                 85876 non-null  object\n",
+      " 5   feeAmount                      85876 non-null  object\n",
+      " 6   id                             85876 non-null  object\n",
+      " 7   oldOutcomeTokenMarginalPrice   85876 non-null  object\n",
+      " 8   outcomeIndex                   85876 non-null  object\n",
+      " 9   outcomeTokenMarginalPrice      85876 non-null  object\n",
+      " 10  outcomeTokensTraded            85876 non-null  object\n",
+      " 11  title                          85876 non-null  object\n",
+      " 12  transactionHash                85876 non-null  object\n",
+      " 13  type                           85876 non-null  object\n",
+      " 14  market_creator                 85876 non-null  object\n",
+      " 15  fpmm.answerFinalizedTimestamp  54399 non-null  object\n",
+      " 16  fpmm.arbitrationOccurred       85876 non-null  bool  \n",
+      " 17  fpmm.currentAnswer             54399 non-null  object\n",
+      " 18  fpmm.id                        85876 non-null  object\n",
+      " 19  fpmm.isPendingArbitration      85876 non-null  bool  \n",
+      " 20  fpmm.openingTimestamp          85876 non-null  object\n",
+      " 21  fpmm.outcomes                  85876 non-null  object\n",
+      " 22  fpmm.title                     85876 non-null  object\n",
+      " 23  fpmm.condition.id              85876 non-null  object\n",
       "dtypes: bool(2), object(22)\n",
-      "memory usage: 10.4+ MB\n"
+      "memory usage: 14.6+ MB\n"
      ]
     }
    ],
@@ -87,7 +95,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
@@ -105,7 +113,7 @@
        "      dtype='object')"
       ]
      },
-     "execution_count": 5,
+     "execution_count": 6,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -116,20 +124,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "4686"
-      ]
-     },
-     "execution_count": 4,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "len(markets.id.unique())"
    ]
@@ -240,7 +237,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -249,7 +246,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
@@ -299,123 +296,123 @@
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
-       "      <td>450426474650738688</td>\n",
-       "      <td>0.4504269694034145716308073094168006</td>\n",
+       "      <td>930596765045617408</td>\n",
+       "      <td>0.9305977993411753386434828033666473</td>\n",
        "      <td>0xe91d153e0b41518a2ce8dd3d7944fa863463a97d</td>\n",
-       "      <td>1724553455</td>\n",
-       "      <td>0x022b36c50b85b8ae7addfb8a35d76c59d5814834</td>\n",
-       "      <td>9008529493014773</td>\n",
-       "      <td>0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f50x02...</td>\n",
-       "      <td>0.592785210609610270634125335572129</td>\n",
-       "      <td>1</td>\n",
-       "      <td>0.6171295391012242250994586583534301</td>\n",
+       "      <td>1728596605</td>\n",
+       "      <td>0x01274796ce41aa8e8312e05a427ffb4b0d2148f6</td>\n",
+       "      <td>9305967650456174</td>\n",
+       "      <td>0x007068173910cf8719b6f2e66a18b6825c9dde820x01...</td>\n",
+       "      <td>0.5581119797629801968338361802418564</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.611825749650855211231211687533889</td>\n",
        "      <td>...</td>\n",
        "      <td>quickstart</td>\n",
-       "      <td>1725071760</td>\n",
+       "      <td>1728822710</td>\n",
        "      <td>False</td>\n",
        "      <td>0x00000000000000000000000000000000000000000000...</td>\n",
-       "      <td>0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f5</td>\n",
+       "      <td>0x007068173910cf8719b6f2e66a18b6825c9dde82</td>\n",
        "      <td>False</td>\n",
-       "      <td>1724976000</td>\n",
+       "      <td>1728691200</td>\n",
        "      <td>[Yes, No]</td>\n",
-       "      <td>Will the first floating offshore wind research...</td>\n",
-       "      <td>0x0e940f12f30e928e4879c52d065d9da739a3d3f020d1...</td>\n",
+       "      <td>Will the emergency public warning tests planne...</td>\n",
+       "      <td>0xa610166e379c42404bd27bf12a16119fdb5171990c3e...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
-       "      <td>610163214546941400</td>\n",
-       "      <td>0.6101636232215150135654007337015298</td>\n",
+       "      <td>1033247234796193800</td>\n",
+       "      <td>1.033250126003339493791032993674525</td>\n",
        "      <td>0xe91d153e0b41518a2ce8dd3d7944fa863463a97d</td>\n",
-       "      <td>1724811940</td>\n",
+       "      <td>1728505575</td>\n",
        "      <td>0x034c4ad84f7ac6638bf19300d5bbe7d9b981e736</td>\n",
-       "      <td>12203264290938828</td>\n",
-       "      <td>0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f50x03...</td>\n",
-       "      <td>0.842992636523755061934822129394812</td>\n",
-       "      <td>1</td>\n",
-       "      <td>0.8523396372892128845826889719620915</td>\n",
+       "      <td>10332472347961938</td>\n",
+       "      <td>0x007068173910cf8719b6f2e66a18b6825c9dde820x03...</td>\n",
+       "      <td>0.6602089902983034451244461308090707</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.7034159692833852946883644485233207</td>\n",
        "      <td>...</td>\n",
        "      <td>quickstart</td>\n",
-       "      <td>1725071760</td>\n",
+       "      <td>1728822710</td>\n",
        "      <td>False</td>\n",
        "      <td>0x00000000000000000000000000000000000000000000...</td>\n",
-       "      <td>0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f5</td>\n",
+       "      <td>0x007068173910cf8719b6f2e66a18b6825c9dde82</td>\n",
        "      <td>False</td>\n",
-       "      <td>1724976000</td>\n",
+       "      <td>1728691200</td>\n",
        "      <td>[Yes, No]</td>\n",
-       "      <td>Will the first floating offshore wind research...</td>\n",
-       "      <td>0x0e940f12f30e928e4879c52d065d9da739a3d3f020d1...</td>\n",
+       "      <td>Will the emergency public warning tests planne...</td>\n",
+       "      <td>0xa610166e379c42404bd27bf12a16119fdb5171990c3e...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
-       "      <td>789065092332460672</td>\n",
-       "      <td>0.7890644120527324071908793822796086</td>\n",
+       "      <td>1206692368842898300</td>\n",
+       "      <td>1.206691596248187968367063717078884</td>\n",
        "      <td>0xe91d153e0b41518a2ce8dd3d7944fa863463a97d</td>\n",
-       "      <td>1724815755</td>\n",
-       "      <td>0x09e9d42a029e8b0c2df3871709a762117a681d92</td>\n",
-       "      <td>15781301846649213</td>\n",
-       "      <td>0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f50x09...</td>\n",
-       "      <td>0.7983775743712442891104598770339028</td>\n",
+       "      <td>1728562895</td>\n",
+       "      <td>0x05e8bbdb89c84a14d05194bbbae81caf2340db72</td>\n",
+       "      <td>12066923688428983</td>\n",
+       "      <td>0x007068173910cf8719b6f2e66a18b6825c9dde820x05...</td>\n",
+       "      <td>0.1931459183043721864309864210684546</td>\n",
        "      <td>1</td>\n",
-       "      <td>0.8152123711444691659642000374025623</td>\n",
+       "      <td>0.3033804066591317111055858533563476</td>\n",
        "      <td>...</td>\n",
        "      <td>quickstart</td>\n",
-       "      <td>1725071760</td>\n",
+       "      <td>1728822710</td>\n",
        "      <td>False</td>\n",
        "      <td>0x00000000000000000000000000000000000000000000...</td>\n",
-       "      <td>0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f5</td>\n",
+       "      <td>0x007068173910cf8719b6f2e66a18b6825c9dde82</td>\n",
        "      <td>False</td>\n",
-       "      <td>1724976000</td>\n",
+       "      <td>1728691200</td>\n",
        "      <td>[Yes, No]</td>\n",
-       "      <td>Will the first floating offshore wind research...</td>\n",
-       "      <td>0x0e940f12f30e928e4879c52d065d9da739a3d3f020d1...</td>\n",
+       "      <td>Will the emergency public warning tests planne...</td>\n",
+       "      <td>0xa610166e379c42404bd27bf12a16119fdb5171990c3e...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
-       "      <td>1000000000000000000</td>\n",
-       "      <td>1.000000605383660329048491794939126</td>\n",
+       "      <td>930598203274544384</td>\n",
+       "      <td>0.9305992375717008091217928729793422</td>\n",
        "      <td>0xe91d153e0b41518a2ce8dd3d7944fa863463a97d</td>\n",
-       "      <td>1724546620</td>\n",
-       "      <td>0x09e9d42a029e8b0c2df3871709a762117a681d92</td>\n",
-       "      <td>20000000000000000</td>\n",
-       "      <td>0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f50x09...</td>\n",
-       "      <td>0.5110745907733438805447072252622708</td>\n",
-       "      <td>1</td>\n",
-       "      <td>0.5746805204222762335911904727318937</td>\n",
+       "      <td>1728596645</td>\n",
+       "      <td>0x17c17ca981b7e244d0bad80b632a082dc1db36e5</td>\n",
+       "      <td>9305982032745443</td>\n",
+       "      <td>0x007068173910cf8719b6f2e66a18b6825c9dde820x17...</td>\n",
+       "      <td>0.611825749650855211231211687533889</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.6579972404391247884756597316198778</td>\n",
        "      <td>...</td>\n",
        "      <td>quickstart</td>\n",
-       "      <td>1725071760</td>\n",
+       "      <td>1728822710</td>\n",
        "      <td>False</td>\n",
        "      <td>0x00000000000000000000000000000000000000000000...</td>\n",
-       "      <td>0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f5</td>\n",
+       "      <td>0x007068173910cf8719b6f2e66a18b6825c9dde82</td>\n",
        "      <td>False</td>\n",
-       "      <td>1724976000</td>\n",
+       "      <td>1728691200</td>\n",
        "      <td>[Yes, No]</td>\n",
-       "      <td>Will the first floating offshore wind research...</td>\n",
-       "      <td>0x0e940f12f30e928e4879c52d065d9da739a3d3f020d1...</td>\n",
+       "      <td>Will the emergency public warning tests planne...</td>\n",
+       "      <td>0xa610166e379c42404bd27bf12a16119fdb5171990c3e...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
-       "      <td>100000000000000000</td>\n",
-       "      <td>0.1000004271262862419547394646567906</td>\n",
+       "      <td>1798695965102918400</td>\n",
+       "      <td>1.798696795931342313936125782275225</td>\n",
        "      <td>0xe91d153e0b41518a2ce8dd3d7944fa863463a97d</td>\n",
-       "      <td>1724771260</td>\n",
-       "      <td>0x0d049dcaece0ecb6fc81a460da7bcc2a4785d6e5</td>\n",
-       "      <td>2000000000000000</td>\n",
-       "      <td>0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f50x0d...</td>\n",
-       "      <td>0.2713968218662319388988681987389408</td>\n",
+       "      <td>1728337780</td>\n",
+       "      <td>0x1d942103400c1f1657dcbffd5e08904787ea936b</td>\n",
+       "      <td>17986959651029184</td>\n",
+       "      <td>0x007068173910cf8719b6f2e66a18b6825c9dde820x1d...</td>\n",
+       "      <td>0.7636157369419787681755577286755703</td>\n",
        "      <td>0</td>\n",
-       "      <td>0.2804586217805511523845593360379658</td>\n",
+       "      <td>0.8080447772492735383356100969932859</td>\n",
        "      <td>...</td>\n",
        "      <td>quickstart</td>\n",
-       "      <td>1725071760</td>\n",
+       "      <td>1728822710</td>\n",
        "      <td>False</td>\n",
        "      <td>0x00000000000000000000000000000000000000000000...</td>\n",
-       "      <td>0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f5</td>\n",
+       "      <td>0x007068173910cf8719b6f2e66a18b6825c9dde82</td>\n",
        "      <td>False</td>\n",
-       "      <td>1724976000</td>\n",
+       "      <td>1728691200</td>\n",
        "      <td>[Yes, No]</td>\n",
-       "      <td>Will the first floating offshore wind research...</td>\n",
-       "      <td>0x0e940f12f30e928e4879c52d065d9da739a3d3f020d1...</td>\n",
+       "      <td>Will the emergency public warning tests planne...</td>\n",
+       "      <td>0xa610166e379c42404bd27bf12a16119fdb5171990c3e...</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
@@ -424,53 +421,53 @@
       ],
       "text/plain": [
        "      collateralAmount                   collateralAmountUSD  \\\n",
-       "0   450426474650738688  0.4504269694034145716308073094168006   \n",
-       "1   610163214546941400  0.6101636232215150135654007337015298   \n",
-       "2   789065092332460672  0.7890644120527324071908793822796086   \n",
-       "3  1000000000000000000   1.000000605383660329048491794939126   \n",
-       "4   100000000000000000  0.1000004271262862419547394646567906   \n",
+       "0   930596765045617408  0.9305977993411753386434828033666473   \n",
+       "1  1033247234796193800   1.033250126003339493791032993674525   \n",
+       "2  1206692368842898300   1.206691596248187968367063717078884   \n",
+       "3   930598203274544384  0.9305992375717008091217928729793422   \n",
+       "4  1798695965102918400   1.798696795931342313936125782275225   \n",
        "\n",
        "                              collateralToken creationTimestamp  \\\n",
-       "0  0xe91d153e0b41518a2ce8dd3d7944fa863463a97d        1724553455   \n",
-       "1  0xe91d153e0b41518a2ce8dd3d7944fa863463a97d        1724811940   \n",
-       "2  0xe91d153e0b41518a2ce8dd3d7944fa863463a97d        1724815755   \n",
-       "3  0xe91d153e0b41518a2ce8dd3d7944fa863463a97d        1724546620   \n",
-       "4  0xe91d153e0b41518a2ce8dd3d7944fa863463a97d        1724771260   \n",
+       "0  0xe91d153e0b41518a2ce8dd3d7944fa863463a97d        1728596605   \n",
+       "1  0xe91d153e0b41518a2ce8dd3d7944fa863463a97d        1728505575   \n",
+       "2  0xe91d153e0b41518a2ce8dd3d7944fa863463a97d        1728562895   \n",
+       "3  0xe91d153e0b41518a2ce8dd3d7944fa863463a97d        1728596645   \n",
+       "4  0xe91d153e0b41518a2ce8dd3d7944fa863463a97d        1728337780   \n",
        "\n",
        "                               trader_address          feeAmount  \\\n",
-       "0  0x022b36c50b85b8ae7addfb8a35d76c59d5814834   9008529493014773   \n",
-       "1  0x034c4ad84f7ac6638bf19300d5bbe7d9b981e736  12203264290938828   \n",
-       "2  0x09e9d42a029e8b0c2df3871709a762117a681d92  15781301846649213   \n",
-       "3  0x09e9d42a029e8b0c2df3871709a762117a681d92  20000000000000000   \n",
-       "4  0x0d049dcaece0ecb6fc81a460da7bcc2a4785d6e5   2000000000000000   \n",
+       "0  0x01274796ce41aa8e8312e05a427ffb4b0d2148f6   9305967650456174   \n",
+       "1  0x034c4ad84f7ac6638bf19300d5bbe7d9b981e736  10332472347961938   \n",
+       "2  0x05e8bbdb89c84a14d05194bbbae81caf2340db72  12066923688428983   \n",
+       "3  0x17c17ca981b7e244d0bad80b632a082dc1db36e5   9305982032745443   \n",
+       "4  0x1d942103400c1f1657dcbffd5e08904787ea936b  17986959651029184   \n",
        "\n",
        "                                                  id  \\\n",
-       "0  0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f50x02...   \n",
-       "1  0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f50x03...   \n",
-       "2  0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f50x09...   \n",
-       "3  0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f50x09...   \n",
-       "4  0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f50x0d...   \n",
+       "0  0x007068173910cf8719b6f2e66a18b6825c9dde820x01...   \n",
+       "1  0x007068173910cf8719b6f2e66a18b6825c9dde820x03...   \n",
+       "2  0x007068173910cf8719b6f2e66a18b6825c9dde820x05...   \n",
+       "3  0x007068173910cf8719b6f2e66a18b6825c9dde820x17...   \n",
+       "4  0x007068173910cf8719b6f2e66a18b6825c9dde820x1d...   \n",
        "\n",
        "           oldOutcomeTokenMarginalPrice outcomeIndex  \\\n",
-       "0   0.592785210609610270634125335572129            1   \n",
-       "1   0.842992636523755061934822129394812            1   \n",
-       "2  0.7983775743712442891104598770339028            1   \n",
-       "3  0.5110745907733438805447072252622708            1   \n",
-       "4  0.2713968218662319388988681987389408            0   \n",
+       "0  0.5581119797629801968338361802418564            0   \n",
+       "1  0.6602089902983034451244461308090707            0   \n",
+       "2  0.1931459183043721864309864210684546            1   \n",
+       "3   0.611825749650855211231211687533889            0   \n",
+       "4  0.7636157369419787681755577286755703            0   \n",
        "\n",
        "              outcomeTokenMarginalPrice  ... market_creator  \\\n",
-       "0  0.6171295391012242250994586583534301  ...     quickstart   \n",
-       "1  0.8523396372892128845826889719620915  ...     quickstart   \n",
-       "2  0.8152123711444691659642000374025623  ...     quickstart   \n",
-       "3  0.5746805204222762335911904727318937  ...     quickstart   \n",
-       "4  0.2804586217805511523845593360379658  ...     quickstart   \n",
+       "0   0.611825749650855211231211687533889  ...     quickstart   \n",
+       "1  0.7034159692833852946883644485233207  ...     quickstart   \n",
+       "2  0.3033804066591317111055858533563476  ...     quickstart   \n",
+       "3  0.6579972404391247884756597316198778  ...     quickstart   \n",
+       "4  0.8080447772492735383356100969932859  ...     quickstart   \n",
        "\n",
        "  fpmm.answerFinalizedTimestamp fpmm.arbitrationOccurred  \\\n",
-       "0                    1725071760                    False   \n",
-       "1                    1725071760                    False   \n",
-       "2                    1725071760                    False   \n",
-       "3                    1725071760                    False   \n",
-       "4                    1725071760                    False   \n",
+       "0                    1728822710                    False   \n",
+       "1                    1728822710                    False   \n",
+       "2                    1728822710                    False   \n",
+       "3                    1728822710                    False   \n",
+       "4                    1728822710                    False   \n",
        "\n",
        "                                  fpmm.currentAnswer  \\\n",
        "0  0x00000000000000000000000000000000000000000000...   \n",
@@ -480,37 +477,37 @@
        "4  0x00000000000000000000000000000000000000000000...   \n",
        "\n",
        "                                      fpmm.id fpmm.isPendingArbitration  \\\n",
-       "0  0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f5                     False   \n",
-       "1  0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f5                     False   \n",
-       "2  0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f5                     False   \n",
-       "3  0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f5                     False   \n",
-       "4  0x0017cd58d6a7ee1451388c7d5b1051b4c0a041f5                     False   \n",
+       "0  0x007068173910cf8719b6f2e66a18b6825c9dde82                     False   \n",
+       "1  0x007068173910cf8719b6f2e66a18b6825c9dde82                     False   \n",
+       "2  0x007068173910cf8719b6f2e66a18b6825c9dde82                     False   \n",
+       "3  0x007068173910cf8719b6f2e66a18b6825c9dde82                     False   \n",
+       "4  0x007068173910cf8719b6f2e66a18b6825c9dde82                     False   \n",
        "\n",
        "   fpmm.openingTimestamp fpmm.outcomes  \\\n",
-       "0             1724976000     [Yes, No]   \n",
-       "1             1724976000     [Yes, No]   \n",
-       "2             1724976000     [Yes, No]   \n",
-       "3             1724976000     [Yes, No]   \n",
-       "4             1724976000     [Yes, No]   \n",
+       "0             1728691200     [Yes, No]   \n",
+       "1             1728691200     [Yes, No]   \n",
+       "2             1728691200     [Yes, No]   \n",
+       "3             1728691200     [Yes, No]   \n",
+       "4             1728691200     [Yes, No]   \n",
        "\n",
        "                                          fpmm.title  \\\n",
-       "0  Will the first floating offshore wind research...   \n",
-       "1  Will the first floating offshore wind research...   \n",
-       "2  Will the first floating offshore wind research...   \n",
-       "3  Will the first floating offshore wind research...   \n",
-       "4  Will the first floating offshore wind research...   \n",
+       "0  Will the emergency public warning tests planne...   \n",
+       "1  Will the emergency public warning tests planne...   \n",
+       "2  Will the emergency public warning tests planne...   \n",
+       "3  Will the emergency public warning tests planne...   \n",
+       "4  Will the emergency public warning tests planne...   \n",
        "\n",
        "                                   fpmm.condition.id  \n",
-       "0  0x0e940f12f30e928e4879c52d065d9da739a3d3f020d1...  \n",
-       "1  0x0e940f12f30e928e4879c52d065d9da739a3d3f020d1...  \n",
-       "2  0x0e940f12f30e928e4879c52d065d9da739a3d3f020d1...  \n",
-       "3  0x0e940f12f30e928e4879c52d065d9da739a3d3f020d1...  \n",
-       "4  0x0e940f12f30e928e4879c52d065d9da739a3d3f020d1...  \n",
+       "0  0xa610166e379c42404bd27bf12a16119fdb5171990c3e...  \n",
+       "1  0xa610166e379c42404bd27bf12a16119fdb5171990c3e...  \n",
+       "2  0xa610166e379c42404bd27bf12a16119fdb5171990c3e...  \n",
+       "3  0xa610166e379c42404bd27bf12a16119fdb5171990c3e...  \n",
+       "4  0xa610166e379c42404bd27bf12a16119fdb5171990c3e...  \n",
        "\n",
        "[5 rows x 24 columns]"
       ]
      },
-     "execution_count": 7,
+     "execution_count": 9,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1107,7 +1104,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 75,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [
     {
@@ -1139,92 +1136,21 @@
        "      <th>first_outcome_prob</th>\n",
        "      <th>second_outcome_prob</th>\n",
        "      <th>kl_divergence</th>\n",
+       "      <th>off_by_perc</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>yes</td>\n",
-       "      <td>0x67490193504b49a247d6a3ba7d441e9894d9615f</td>\n",
-       "      <td>1722470400</td>\n",
-       "      <td>quickstart</td>\n",
-       "      <td>2024-08-01 02:00:00</td>\n",
-       "      <td>0.8145</td>\n",
-       "      <td>0.1855</td>\n",
-       "      <td>3.791664</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>no</td>\n",
-       "      <td>0x17f2c97bf52a79671878201bf2995a3b6daba041</td>\n",
-       "      <td>1722470400</td>\n",
-       "      <td>quickstart</td>\n",
-       "      <td>2024-08-01 02:00:00</td>\n",
-       "      <td>0.1975</td>\n",
-       "      <td>0.8025</td>\n",
-       "      <td>4.050688</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2</th>\n",
-       "      <td>no</td>\n",
-       "      <td>0xbca6aa704a02a5c5a766ff829dacc81aee5547cf</td>\n",
-       "      <td>1722470400</td>\n",
-       "      <td>quickstart</td>\n",
-       "      <td>2024-08-01 02:00:00</td>\n",
-       "      <td>0.6969</td>\n",
-       "      <td>0.3031</td>\n",
-       "      <td>15.433247</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3</th>\n",
-       "      <td>no</td>\n",
-       "      <td>0x221c71bab604691b0b8805c1c433fc8e22123a67</td>\n",
-       "      <td>1722470400</td>\n",
-       "      <td>pearl</td>\n",
-       "      <td>2024-08-01 02:00:00</td>\n",
-       "      <td>0.4757</td>\n",
-       "      <td>0.5243</td>\n",
-       "      <td>10.261432</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>4</th>\n",
-       "      <td>no</td>\n",
-       "      <td>0xe4d078b9be12319c0063f58dc10f19604a5df163</td>\n",
-       "      <td>1722470400</td>\n",
-       "      <td>quickstart</td>\n",
-       "      <td>2024-08-01 02:00:00</td>\n",
-       "      <td>0.3473</td>\n",
-       "      <td>0.6527</td>\n",
-       "      <td>7.351119</td>\n",
-       "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "text/plain": [
-       "  currentAnswer                                          id openingTimestamp  \\\n",
-       "0           yes  0x67490193504b49a247d6a3ba7d441e9894d9615f       1722470400   \n",
-       "1            no  0x17f2c97bf52a79671878201bf2995a3b6daba041       1722470400   \n",
-       "2            no  0xbca6aa704a02a5c5a766ff829dacc81aee5547cf       1722470400   \n",
-       "3            no  0x221c71bab604691b0b8805c1c433fc8e22123a67       1722470400   \n",
-       "4            no  0xe4d078b9be12319c0063f58dc10f19604a5df163       1722470400   \n",
-       "\n",
-       "  market_creator    opening_datetime  first_outcome_prob  second_outcome_prob  \\\n",
-       "0     quickstart 2024-08-01 02:00:00              0.8145               0.1855   \n",
-       "1     quickstart 2024-08-01 02:00:00              0.1975               0.8025   \n",
-       "2     quickstart 2024-08-01 02:00:00              0.6969               0.3031   \n",
-       "3          pearl 2024-08-01 02:00:00              0.4757               0.5243   \n",
-       "4     quickstart 2024-08-01 02:00:00              0.3473               0.6527   \n",
-       "\n",
-       "   kl_divergence  \n",
-       "0       3.791664  \n",
-       "1       4.050688  \n",
-       "2      15.433247  \n",
-       "3      10.261432  \n",
-       "4       7.351119  "
+       "Empty DataFrame\n",
+       "Columns: [currentAnswer, id, openingTimestamp, market_creator, opening_datetime, first_outcome_prob, second_outcome_prob, kl_divergence, off_by_perc]\n",
+       "Index: []"
       ]
      },
-     "execution_count": 75,
+     "execution_count": 10,
      "metadata": {},
      "output_type": "execute_result"
     }

notebooks/daily_data.ipynb

@@ -231,33 +231,33 @@
      "output_type": "stream",
      "text": [
       "<class 'pandas.core.frame.DataFrame'>\n",
-      "RangeIndex: 3882 entries, 0 to 3881\n",
+      "RangeIndex: 9841 entries, 0 to 9840\n",
       "Data columns (total 21 columns):\n",
       " #   Column                  Non-Null Count  Dtype              \n",
       "---  ------                  --------------  -----              \n",
-      " 0   trader_address          3882 non-null   object             \n",
-      " 1   market_creator          3882 non-null   object             \n",
-      " 2   trade_id                3882 non-null   object             \n",
-      " 3   creation_timestamp      3882 non-null   datetime64[ns, UTC]\n",
-      " 4   title                   3882 non-null   object             \n",
-      " 5   market_status           3882 non-null   object             \n",
-      " 6   collateral_amount       3882 non-null   float64            \n",
-      " 7   outcome_index           3882 non-null   object             \n",
-      " 8   trade_fee_amount        3882 non-null   float64            \n",
-      " 9   outcomes_tokens_traded  3882 non-null   float64            \n",
-      " 10  current_answer          0 non-null      object             \n",
-      " 11  is_invalid              3882 non-null   bool               \n",
-      " 12  winning_trade           0 non-null      object             \n",
-      " 13  earnings                3882 non-null   float64            \n",
-      " 14  redeemed                3882 non-null   bool               \n",
-      " 15  redeemed_amount         3882 non-null   int64              \n",
-      " 16  num_mech_calls          3882 non-null   int64              \n",
-      " 17  mech_fee_amount         3882 non-null   float64            \n",
-      " 18  net_earnings            3882 non-null   float64            \n",
-      " 19  roi                     3882 non-null   float64            \n",
-      " 20  staking                 3882 non-null   object             \n",
-      "dtypes: bool(2), datetime64[ns, UTC](1), float64(7), int64(2), object(9)\n",
-      "memory usage: 583.9+ KB\n"
+      " 0   trader_address          9841 non-null   object             \n",
+      " 1   market_creator          9841 non-null   object             \n",
+      " 2   trade_id                9841 non-null   object             \n",
+      " 3   creation_timestamp      9841 non-null   datetime64[ns, UTC]\n",
+      " 4   title                   9841 non-null   object             \n",
+      " 5   market_status           9841 non-null   object             \n",
+      " 6   collateral_amount       9841 non-null   float64            \n",
+      " 7   outcome_index           9841 non-null   object             \n",
+      " 8   trade_fee_amount        9841 non-null   float64            \n",
+      " 9   outcomes_tokens_traded  9841 non-null   float64            \n",
+      " 10  current_answer          1080 non-null   float64            \n",
+      " 11  is_invalid              9841 non-null   bool               \n",
+      " 12  winning_trade           1080 non-null   object             \n",
+      " 13  earnings                9841 non-null   float64            \n",
+      " 14  redeemed                9841 non-null   bool               \n",
+      " 15  redeemed_amount         9841 non-null   int64              \n",
+      " 16  num_mech_calls          9841 non-null   int64              \n",
+      " 17  mech_fee_amount         9841 non-null   float64            \n",
+      " 18  net_earnings            9841 non-null   float64            \n",
+      " 19  roi                     9841 non-null   float64            \n",
+      " 20  staking                 9841 non-null   object             \n",
+      "dtypes: bool(2), datetime64[ns, UTC](1), float64(8), int64(2), object(8)\n",
+      "memory usage: 1.4+ MB\n"
      ]
     }
    ],
@@ -293,16 +293,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 23,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "Timestamp('2024-11-26 10:19:30+0000', tz='UTC')"
+       "Timestamp('2024-11-28 13:48:40+0000', tz='UTC')"
       ]
      },
-     "execution_count": 23,
+     "execution_count": 5,
      "metadata": {},
      "output_type": "execute_result"
     }