updating KL-div formula

data/closed_markets_div.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8d2583a54d7d4b38e51c2bf1e808a710e6a4b232009b5ae8917ce0534b045d94
-size 48882
+oid sha256:abdc271bdfd214e6497de73c32adeada50b680d847b1ad9b0bdf2c2fc442394b
+size 48595

notebooks/closed_markets.ipynb

@@ -1468,7 +1468,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 31,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -1487,6 +1487,7 @@
     "    # Avoid division by zero\n",
     "    epsilon = 1e-10\n",
     "    q = np.clip(q, epsilon, 1 - epsilon)\n",
+    "    print(q)\n",
     "\n",
     "    # Compute KL divergence\n",
     "    kl_div = np.sum(p * np.log(p / q))\n",
@@ -1494,6 +1495,72 @@
     "    return kl_div"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([-22.82067008,   1.6847004 ])"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "p= 0\n",
+    "q = 0.8145\n",
+    "p = np.array([p, 1 - p])\n",
+    "q = np.array([q, 1 - q])\n",
+    "epsilon = 1e-10\n",
+    "p = np.clip(p, epsilon, 1 - epsilon)\n",
+    "np.log(p/q)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([-2.28206701e-09,  1.68470040e+00])"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "p*np.log(p/q)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "1.6847003943841101"
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "np.sum(p * np.log(p / q))"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 3,
@@ -1518,7 +1585,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [
     {
@@ -1527,7 +1594,7 @@
        "0.16397451204513597"
       ]
      },
-     "execution_count": 6,
+     "execution_count": 12,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1561,6 +1628,347 @@
     "kl = np.sum(kl_div(p, q))\n",
     "print(f\"KL divergence: {kl}\")"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0.2051808486854041"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "p= 1\n",
+    "q = 0.8145\n",
+    "kl_divergence(p, q)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "KL divergence: 0.20518085094003724\n"
+     ]
+    }
+   ],
+   "source": [
+    "# For multiple probabilities\n",
+    "p = np.array([1.0, 0.0])\n",
+    "q = np.array([0.8145, 0.1855])\n",
+    "\n",
+    "kl = np.sum(kl_div(p, q))\n",
+    "print(f\"KL divergence: {kl}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[1.e+00 1.e-10]\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/var/folders/gp/02mb1d514ng739czlxw1lhh00000gn/T/ipykernel_28964/3714966623.py:19: RuntimeWarning: divide by zero encountered in log\n",
+      "  kl_div = np.sum(p * np.log(p / q))\n",
+      "/var/folders/gp/02mb1d514ng739czlxw1lhh00000gn/T/ipykernel_28964/3714966623.py:19: RuntimeWarning: invalid value encountered in multiply\n",
+      "  kl_div = np.sum(p * np.log(p / q))\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "nan"
+      ]
+     },
+     "execution_count": 32,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "p = 0\n",
+    "q = 1\n",
+    "kl_divergence(p, q)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "KL divergence: 0.010050335853501449\n"
+     ]
+    }
+   ],
+   "source": [
+    "p = np.array([0.0, 1.0])\n",
+    "q = np.array([0.01, 0.99])\n",
+    "\n",
+    "kl = np.sum(kl_div(p, q))\n",
+    "print(f\"KL divergence: {kl}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0.17425697504355725"
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "p = 0.01\n",
+    "q = 0\n",
+    "kl_divergence(p, q)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "KL divergence: inf\n"
+     ]
+    }
+   ],
+   "source": [
+    "p = np.array([0.01, 0.99])\n",
+    "q = np.array([0.0, 1.0])\n",
+    "\n",
+    "kl = np.sum(kl_div(p, q))\n",
+    "print(f\"KL divergence: {kl}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([0.01, 0.99])"
+      ]
+     },
+     "execution_count": 25,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "epsilon = 1e-10\n",
+    "q = 0\n",
+    "q = np.clip(p, epsilon, 1 - epsilon)\n",
+    "q"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# New function"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 47,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def kl_divergence(P, Q):\n",
+    "    \"\"\"\n",
+    "    Compute KL divergence for a single sample with two prob distributions.\n",
+    "\n",
+    "    :param P: True distribution)\n",
+    "    :param Q: Approximating distribution)\n",
+    "    :return: KL divergence value\n",
+    "    \"\"\"\n",
+    "    # Review edge cases\n",
+    "    if P[0] == Q[0]:\n",
+    "        return 0.0\n",
+    "    # If P is complete opposite of Q, divergence is some max value.\n",
+    "    # Here set to 20--allows for Q [\\mu, 1-\\mu] or Q[1-\\mu, \\mu] where \\mu = 10^-8\n",
+    "    if P[0] == Q[1]:\n",
+    "        return 20\n",
+    "\n",
+    "    nonzero = P > 0.0\n",
+    "    # Compute KL divergence\n",
+    "    kl_div = np.sum(P[nonzero] * np.log(P[nonzero] / Q[nonzero]))\n",
+    "\n",
+    "    return kl_div"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 45,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "P = np.array([0.0, 1.0])\n",
+    "Q = np.array([0.0, 1.0])\n",
+    "print(kl_divergence(P,Q))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 46,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "20\n"
+     ]
+    }
+   ],
+   "source": [
+    "P = np.array([0.0, 1.0])\n",
+    "Q = np.array([1.0, 0.0])\n",
+    "print(kl_divergence(P,Q))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "20\n"
+     ]
+    }
+   ],
+   "source": [
+    "P = np.array([1.0, 0.0])\n",
+    "Q = np.array([0.0, 1.0])\n",
+    "print(kl_divergence(P,Q))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 49,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0.010050335853501506\n"
+     ]
+    }
+   ],
+   "source": [
+    "P = np.array([0.0, 1.0])\n",
+    "Q = np.array([0.01, 0.99])\n",
+    "print(kl_divergence(P,Q))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0.010050335853501506"
+      ]
+     },
+     "execution_count": 43,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "1 * np.log(1 / 0.99)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 48,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "4.605170185988092\n"
+     ]
+    }
+   ],
+   "source": [
+    "P = np.array([1.0, 0.0])\n",
+    "Q = np.array([0.01, 0.99])\n",
+    "print(kl_divergence(P,Q))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 44,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "4.605170185988092"
+      ]
+     },
+     "execution_count": 44,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "np.log(1/0.01)"
+   ]
   }
  ],
  "metadata": {

scripts/closed_markets_divergence.py

@@ -138,24 +138,25 @@ def get_closed_markets():
     return markets
 
 
-def kl_divergence(p, q):
+def kl_divergence(P, Q):
     """
-    Compute KL divergence for a single sample with two probabilities.
+    Compute KL divergence for a single sample with two prob distributions.
 
-    :param p: First probability (true distribution)
-    :param q: Second probability (approximating distribution)
+    :param P: True distribution)
+    :param Q: Approximating distribution)
     :return: KL divergence value
     """
-    # Ensure probabilities sum to 1
-    p = np.array([p, 1 - p])
-    q = np.array([q, 1 - q])
-
-    # Avoid division by zero
-    epsilon = 1e-10
-    q = np.clip(q, epsilon, 1 - epsilon)
+    # Review edge cases
+    if P[0] == Q[0]:
+        return 0.0
+    # If P is complete opposite of Q, divergence is some max value.
+    # Here set to 20--allows for Q [\mu, 1-\mu] or Q[1-\mu, \mu] where \mu = 10^-8
+    if P[0] == Q[1]:
+        return 20
 
+    nonzero = P > 0.0
     # Compute KL divergence
-    kl_div = np.sum(p * np.log(p / q))
+    kl_div = np.sum(P[nonzero] * np.log(P[nonzero] / Q[nonzero]))
 
     return kl_div
 
@@ -165,12 +166,15 @@ def market_KL_divergence(market_row: pd.DataFrame) -> float:
     Formula in https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence"""
     current_answer = market_row.currentAnswer  # "yes", "no"
     approx_prob = market_row.first_outcome_prob
-    true_prob = 0.99  # for yes outcome (99% is the max we can specify to avoid nan)
+    true_prob = 1.0  # for yes outcome
     if current_answer == "no":
-        true_prob = 0.01  # = 0.1% for yes outcome and 99% for no
+        true_prob = 0.0  # = 0% for yes outcome and 100% for no
 
     # we have only one sample, the final probability based on tokens
-    return kl_divergence(true_prob, approx_prob)
+    # Ensure probabilities sum to 1
+    P = np.array([true_prob, 1 - true_prob])
+    Q = np.array([approx_prob, 1 - approx_prob])
+    return kl_divergence(P, Q)
 
 
 def compute_tokens_prob(token_amounts: list) -> list: