Use Evalica

README.md

@@ -13,4 +13,11 @@ license: apache-2.0
 
 # Pair2Rank
 
-It's all about [ranking](https://arxiv.org/abs/2207.00076).
+This spaces uses the [Evalica](https://github.com/dustalov/evalica) library for pairwise comparisons, exposing the following methods:
+
+- Counting
+- [Bradley-Terry (1952)](https://doi.org/10.2307/2334029)
+- [Elo (1960)](https://web.archive.org/web/20080926015601/http://www.uschess.org/about/about.php)
+- [Eigenvector (1987)](https://doi.org/10.1086/228631)
+- [PageRank (1998)](https://doi.org/10.1016/S0169-7552(98)00110-X)
+- [Newman (2023)]((https://arxiv.org/abs/2207.00076))

app.py

@@ -17,19 +17,18 @@
 __author__ = 'Dmitry Ustalov'
 __license__ = 'Apache 2.0'
 
-from collections.abc import Callable
-from functools import partial
 from typing import BinaryIO, cast
 
+import evalica
 import gradio as gr
 import networkx as nx
 import numpy as np
-import numpy.typing as npt
 import pandas as pd
 import plotly.express as px
+from evalica import Winner
 from plotly.graph_objects import Figure
 
-TOLERANCE, LIMIT = 1e-16, 1000
+TOLERANCE, LIMIT = 1e-6, 100
 
 
 def visualize(df_pairwise: pd.DataFrame) -> Figure:
@@ -39,134 +38,56 @@ def visualize(df_pairwise: pd.DataFrame) -> Figure:
     return fig
 
 
-# https://gist.github.com/dustalov/41678b70c40ba5a55430fa5e77b121d9#file-bradley_terry-py
-def bradley_terry(wins: npt.NDArray[np.int64], ties: npt.NDArray[np.int64]) -> npt.NDArray[np.float64]:
-    M = wins + .5 * ties
+def counting(xs: list[str], ys: list[str], ws: list[Winner]) -> tuple["pd.Series[str]", "pd.Index[str]"]:
+    result = evalica.counting(xs, ys, ws)
+    return result.scores, result.index
 
-    T = M.T + M
-    active = T > 0
 
-    w = M.sum(axis=1)
+def bradley_terry(xs: list[str], ys: list[str], ws: list[Winner]) -> tuple["pd.Series[str]", "pd.Index[str]"]:
+    result = evalica.bradley_terry(xs, ys, ws, tolerance=TOLERANCE, limit=LIMIT)
+    return result.scores, result.index
 
-    Z = np.zeros_like(M, dtype=float)
 
-    p = np.ones(M.shape[0])
-    p_new = p.copy()
+def elo(xs: list[str], ys: list[str], ws: list[Winner]) -> tuple["pd.Series[str]", "pd.Index[str]"]:
+    result = evalica.elo(xs, ys, ws)
+    return result.scores, result.index
 
-    converged, iterations = False, 0
 
-    while not converged:
-        iterations += 1
+def eigen(xs: list[str], ys: list[str], ws: list[Winner]) -> tuple["pd.Series[str]", "pd.Index[str]"]:
+    result = evalica.eigen(xs, ys, ws, tolerance=TOLERANCE, limit=LIMIT)
+    return result.scores, result.index
 
-        P = np.broadcast_to(p, M.shape)
 
-        Z[active] = T[active] / (P[active] + P.T[active])
+def pagerank(xs: list[str], ys: list[str], ws: list[Winner]) -> tuple["pd.Series[str]", "pd.Index[str]"]:
+    result = evalica.pagerank(xs, ys, ws, tolerance=TOLERANCE, limit=LIMIT)
+    return result.scores, result.index
 
-        p_new[:] = w
-        p_new /= Z.sum(axis=0)
-        p_new /= p_new.sum()
 
-        converged = bool(np.linalg.norm(p_new - p) < TOLERANCE) or (iterations >= LIMIT)
-
-        p[:] = p_new
-
-    return p
-
-
-def centrality(algorithm: Callable[[nx.DiGraph], dict[int, float]],
-               wins: npt.NDArray[np.int64], ties: npt.NDArray[np.int64]) -> npt.NDArray[np.float64]:
-    A = wins + .5 * ties
-
-    G = nx.from_numpy_array(A, create_using=nx.DiGraph)
-
-    scores: dict[int, float] = algorithm(G)
-
-    p = np.array([scores[i] for i in range(len(G))])
-
-    return p
-
-
-def counting(wins: npt.NDArray[np.int64], ties: npt.NDArray[np.int64]) -> npt.NDArray[np.float64]:
-    M = wins + .5 * ties
-
-    return cast(npt.NDArray[np.float64], M.sum(axis=1))
-
-
-def eigen(wins: npt.NDArray[np.int64], ties: npt.NDArray[np.int64]) -> npt.NDArray[np.float64]:
-    algorithm = partial(nx.algorithms.eigenvector_centrality_numpy, max_iter=LIMIT, tol=TOLERANCE, weight='weight')
-
-    return centrality(algorithm, wins, ties)
-
-
-def pagerank(wins: npt.NDArray[np.int64], ties: npt.NDArray[np.int64]) -> npt.NDArray[np.float64]:
-    algorithm = partial(nx.algorithms.pagerank, max_iter=LIMIT, tol=TOLERANCE, weight='weight')
-
-    return centrality(algorithm, wins, ties)
-
-
-# https://gist.github.com/dustalov/41678b70c40ba5a55430fa5e77b121d9#file-newman-py
-def newman(wins: npt.NDArray[np.int64], ties: npt.NDArray[np.int64]) -> npt.NDArray[np.float64]:
-    pi, v = np.ones(wins.shape[0]), .5
-
-    converged, iterations = False, 0
-
-    while not converged:
-        iterations += 1
-
-        v_numerator = np.sum(
-            ties * (pi[:, np.newaxis] + pi) /
-            (pi[:, np.newaxis] + pi + 2 * v * np.sqrt(pi[:, np.newaxis] * pi))
-        ) / 2
-
-        v_denominator = np.sum(
-            wins * 2 * np.sqrt(pi[:, np.newaxis] * pi) /
-            (pi[:, np.newaxis] + pi + 2 * v * np.sqrt(pi[:, np.newaxis] * pi))
-        )
-
-        v = v_numerator / v_denominator
-        v = np.nan_to_num(v, nan=TOLERANCE)
-
-        pi_old = pi.copy()
-
-        pi_numerator = np.sum(
-            (wins + ties / 2) * (pi + v * np.sqrt(pi[:, np.newaxis] * pi)) /
-            (pi[:, np.newaxis] + pi + 2 * v * np.sqrt(pi[:, np.newaxis] * pi)),
-            axis=1
-        )
-
-        pi_denominator = np.sum(
-            (wins + ties / 2) * (1 + v * np.sqrt(pi[:, np.newaxis] * pi)) /
-            (pi[:, np.newaxis] + pi + 2 * v * np.sqrt(pi[:, np.newaxis] * pi)),
-            axis=0
-        )
-
-        pi = pi_numerator / pi_denominator
-        pi = np.nan_to_num(pi, nan=TOLERANCE)
-
-        converged = np.allclose(pi / (pi + 1), pi_old / (pi_old + 1),
-                                rtol=TOLERANCE, atol=TOLERANCE) or (iterations >= LIMIT)
-
-    return pi
+def newman(xs: list[str], ys: list[str], ws: list[Winner]) -> tuple["pd.Series[str]", "pd.Index[str]"]:
+    result = evalica.newman(xs, ys, ws, tolerance=TOLERANCE, limit=LIMIT)
+    return result.scores, result.index
 
 
 ALGORITHMS = {
     'Counting': counting,
     'Bradley-Terry (1952)': bradley_terry,
-    'Eigenvector (1986)': eigen,
+    'Elo (1960)': elo,
+    'Eigenvector (1987)': eigen,
     'PageRank (1998)': pagerank,
     'Newman (2023)': newman,
 }
 
 
-def largest_strongly_connected_component(df: pd.DataFrame) -> set[str]:
-    G = nx.from_pandas_edgelist(df, source='left', target='right', create_using=nx.DiGraph)
-    H = nx.from_pandas_edgelist(df[df['winner'] == 'tie'], source='right', target='left', create_using=nx.DiGraph)
+def largest_strongly_connected_component(df_pairs: pd.DataFrame) -> set[str]:
+    G = nx.from_pandas_edgelist(df_pairs, source='left', target='right', create_using=nx.DiGraph)
+    H = nx.from_pandas_edgelist(df_pairs[df_pairs['winner'] == 'tie'], source='right', target='left',
+                                create_using=nx.DiGraph)
     F = nx.compose(G, H)
     largest = max(nx.strongly_connected_components(F), key=len)
     return cast(set[str], largest)
 
 
-def handler(file: BinaryIO, algorithm: str, filtered: bool, truncated: bool, seed: int) -> tuple[pd.DataFrame, Figure]:
+def handler(file: BinaryIO, algorithm: str, filtered: bool, truncated: bool) -> tuple[pd.DataFrame, Figure]:
     if file is None:
         raise gr.Error('File must be uploaded')
 
@@ -174,56 +95,37 @@ def handler(file: BinaryIO, algorithm: str, filtered: bool, truncated: bool, see
         raise gr.Error(f'Unknown algorithm: {algorithm}')
 
     try:
-        df = pd.read_csv(file.name, dtype=str)
+        df_pairs = pd.read_csv(file.name, dtype=str)
     except ValueError as e:
         raise gr.Error(f'Parsing error: {e}')
 
-    if not pd.Series(['left', 'right', 'winner']).isin(df.columns).all():
+    if not pd.Series(['left', 'right', 'winner']).isin(df_pairs.columns).all():
         raise gr.Error('Columns must exist: left, right, winner')
 
-    if not df['winner'].isin(pd.Series(['left', 'right', 'tie'])).all():
+    if not df_pairs['winner'].isin(pd.Series(['left', 'right', 'tie'])).all():
         raise gr.Error('Allowed winner values: left, right, tie')
 
-    df = df[['left', 'right', 'winner']]
-
-    df.dropna(axis=0, inplace=True)
+    df_pairs = df_pairs[['left', 'right', 'winner']]
 
-    df.loc[df['winner'] == 'right', ['left', 'right']] = df.loc[df['winner'] == 'right', ['right', 'left']].values
-    df.loc[df['winner'] == 'right', 'winner'] = 'left'
+    df_pairs.dropna(axis=0, inplace=True)
 
     if filtered:
-        largest = largest_strongly_connected_component(df)
-
-        df.drop(df[~(df['left'].isin(largest) & df['right'].isin(largest))].index, inplace=True)
-
-        index = pd.Index(largest, name='item')
-    else:
-        index = pd.Index(np.unique(df[['left', 'right']].values), name='item')
-
-    df_wins = pd.pivot_table(df[df['winner'] != 'tie'],
-                             index='left', columns='right', values='winner',
-                             aggfunc='count', fill_value=0)
-    df_wins = df_wins.reindex(labels=index, columns=index, fill_value=0, copy=False)
+        largest = largest_strongly_connected_component(df_pairs)
 
-    df_ties = pd.pivot_table(df[df['winner'] == 'tie'],
-                             index='left', columns='right', values='winner',
-                             aggfunc='count', fill_value=0)
-    df_ties = df_ties.reindex(labels=index, columns=index, fill_value=0, copy=False)
+        df_pairs.drop(df_pairs[~(df_pairs['left'].isin(largest) & df_pairs['right'].isin(largest))].index, inplace=True)
 
-    wins = df_wins.to_numpy(dtype=int)
-    ties = df_ties.to_numpy(dtype=int)
-    ties += ties.T
+    xs, ys = df_pairs["left"], df_pairs["right"]
+    ws = df_pairs["winner"].map({"left": Winner.X, "right": Winner.Y, "tie": Winner.Draw})
 
-    assert wins.shape == ties.shape, 'wins and ties shapes are different'
-
-    scores = ALGORITHMS[algorithm](wins, ties)
+    scores, index = ALGORITHMS[algorithm](xs, ys, ws)
+    index.name = 'item'
 
     df_result = pd.DataFrame(data={'score': scores}, index=index)
 
     df_result['pairs'] = pd.Series(0, dtype=int, index=index).add(
-        df.groupby('left')['left'].count(), fill_value=0
+        df_pairs.groupby('left')['left'].count(), fill_value=0
     ).add(
-        df.groupby('right')['right'].count(), fill_value=0
+        df_pairs.groupby('right')['right'].count(), fill_value=0
     ).astype(int)
 
     df_result['rank'] = df_result['score'].rank(na_option='bottom', ascending=False).astype(int)
@@ -236,9 +138,9 @@ def handler(file: BinaryIO, algorithm: str, filtered: bool, truncated: bool, see
         df_result = pd.concat((df_result.head(5), df_result.tail(5)), copy=False)
         df_result = df_result[~df_result.index.duplicated(keep='last')]
 
-    df_pairwise = pd.DataFrame(data=scores[:, np.newaxis] / (scores + scores[:, np.newaxis]),
-                               index=index, columns=index)
-    df_pairwise = df_pairwise.reindex(labels=df_result['item'], columns=df_result['item'], copy=False)
+    pairwise = evalica.pairwise_scores(df_result['score'].to_numpy())
+
+    df_pairwise = pd.DataFrame(data=pairwise, index=df_result['item'], columns=df_result['item'])
 
     fig = visualize(df_pairwise)
 
@@ -272,10 +174,6 @@ def main() -> None:
                 info='Perform the entire computation but output only five head and five tail items, '
                      'avoiding overlap.'
             ),
-            gr.Number(
-                label='Seed',
-                precision=0
-            )
         ],
         outputs=[
             gr.Dataframe(
@@ -287,12 +185,13 @@ def main() -> None:
             )
         ],
         examples=[
-            ['food.csv', 'Counting', False, False, 0],
-            ['food.csv', 'Bradley-Terry (1952)', False, False, 0],
-            ['food.csv', 'Eigenvector (1986)', False, False, 0],
-            ['food.csv', 'PageRank (1998)', False, False, 0],
-            ['food.csv', 'Newman (2023)', False, False, 0],
-            ['llmfao.csv', 'Bradley-Terry (1952)', False, True, 0]
+            ['food.csv', 'Counting', False, False],
+            ['food.csv', 'Bradley-Terry (1952)', False, False],
+            ['food.csv', 'Eigenvector (1987)', False, False],
+            ['food.csv', 'PageRank (1998)', False, False],
+            ['food.csv', 'Newman (2023)', False, False],
+            ['llmfao.csv', 'Bradley-Terry (1952)', False, True],
+            ['llmfao.csv', 'Elo (1960)', False, True],
         ],
         title='Pair2Rank: Turn Your Side-by-Side Comparisons into Ranking!',
         description='''
@@ -309,6 +208,8 @@ Possible values for `winner` are `left`, `right`, or `tie`. The provided example
 As the output, this tool provides a table with items, their estimated scores, and ranks.
         '''.strip(),
         article='''
+Pair2Rank uses the [Evalica](https://pypi.org/p/evalica) library for computing the scores: <https://github.com/dustalov/evalica>.
+
 Read more about Pair2Rank at <https://evalovernite.substack.com/p/llmfao-human-ranking>.
         '''.strip(),
         allow_flagging='never'

requirements.txt

@@ -1,3 +1,3 @@
+evalica
 networkx
 plotly
-scipy