benchmarks

src/App.tsx

@@ -1,29 +1,22 @@
-import React, { useState, useEffect } from "react";
+import React, { useState, useEffect, useMemo } from "react";
 import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
-import { Checkbox } from "@/components/ui/checkbox";
 import { Input } from "@/components/ui/input";
-import {
-  Table,
-  TableBody,
-  TableCell,
-  TableHead,
-  TableHeader,
-  TableRow,
-} from "@/components/ui/table";
-import { MultiSelect } from "@/components/ui/multi-select";
-import {
-  Collapsible,
-  CollapsibleContent,
-  CollapsibleTrigger,
-} from "@/components/ui/collapsible";
-import { Button } from "@/components/ui/button";
-import { ChevronDown, ChevronRight } from "lucide-react";
-import { mockData } from "./lib/data"; // Assuming you have this file for mock data
-import { Switch } from "@/components/ui/switch";
-
-interface FlattenedModel extends Model {
+import { mockData } from "@/lib/data";
+
+import { ComparisonSelector } from "@/components/ComparisonSelector";
+import { PricingTable } from "@/components/PricingTable";
+
+import { BenchmarkTable } from "./components/BenchmarkTable";
+import { benchmarkData } from "./lib/benchmarks/ index";
+import { BenchmarkComparisonSelector } from "./components/BenchmarkComparisonSelector";
+import { benchmarkMetricOrder } from "./lib/benchmarks/types";
+
+export interface FlattenedModel extends Model {
   provider: string;
   uri: string;
+  benchmark?: {
+    [key: string]: number;
+  };
 }
 
 export interface Model {
@@ -47,171 +40,194 @@ const App: React.FC = () => {
   const [selectedModels, setSelectedModels] = useState<string[]>([]);
   const [expandedProviders, setExpandedProviders] = useState<string[]>([]);
   const [tokenCalculation, setTokenCalculation] = useState<string>("million");
-  const [linkProviderModel, setLinkProviderModel] = useState<boolean>(false);
+  const [benchmarkComparisonMetrics, setBenchmarkComparisonMetrics] = useState<string[]>([]);
+  const [selectedBenchmarkProviders, setSelectedBenchmarkProviders] = useState<string[]>([]);
+  const [selectedBenchmarkModels, setSelectedBenchmarkModels] = useState<string[]>([]);
 
   const [sortConfig, setSortConfig] = useState<{
     key: keyof FlattenedModel;
     direction: string;
   } | null>(null);
 
+  const [benchmarkSortConfig, setBenchmarkSortConfig] = useState<{
+    key: string;
+    direction: "ascending" | "descending";
+  } | null>(null);
+
+
   useEffect(() => {
     setData(mockData);
   }, []);
 
-  const calculatePrice = (price: number, tokens: number): number => {
-    let multiplier = 1;
-    if (tokenCalculation === "thousand") {
-      multiplier = 1e-3;
-    } else if (tokenCalculation === "unit") {
-      multiplier = 1e-6;
-    } else if (tokenCalculation === "billion") {
-      multiplier = 1e3;
-    }
-    return price * tokens * multiplier;
-  };
-
-  const calculateComparison = (
-    modelPrice: number,
-    comparisonPrice: number
-  ): string => {
-    return (((modelPrice - comparisonPrice) / comparisonPrice) * 100).toFixed(
-      2
-    );
-  };
 
-  const flattenData = (data: Provider[]) => {
-    return data.flatMap((provider) =>
+  const flattenDataFromPricing = (data: Provider[]): FlattenedModel[] =>
+    data.flatMap((provider) =>
       provider.models.map((model) => ({
         provider: provider.provider,
         uri: provider.uri,
         ...model,
+        benchmark: {},
       }))
     );
-  };
 
-  const filteredData =
-    selectedProviders.length === 0 &&
-    selectedModels.length === 0 &&
-    !linkProviderModel
-      ? data.map((provider) => ({
-          ...provider,
-          models: provider.models,
-        }))
-      : data
-          .filter(
-            (provider) =>
-              selectedProviders.length === 0 ||
-              selectedProviders.includes(provider.provider)
-          )
-          .map((provider) => ({
-            ...provider,
-            models: provider.models.filter((model) => {
-              // If linking is enabled and no models are selected, filter by provider
-              if (linkProviderModel && selectedModels.length === 0)
-                return selectedProviders.includes(provider.provider);
-
-              // If no models are selected and linking is off, show all models from selected providers (or all if no providers selected)
-              if (!linkProviderModel && selectedModels.length === 0)
-                return (
-                  selectedProviders.length === 0 ||
-                  selectedProviders.includes(provider.provider)
-                );
-
-              // Otherwise, only show selected models
-              return selectedModels.includes(model.name);
-            }),
-          }))
-          .filter((provider) => provider.models.length > 0);
-
-  const sortedFlattenedData = React.useMemo(() => {
-    let sortableData: FlattenedModel[] = flattenData(filteredData);
-    if (sortConfig !== null) {
-      sortableData.sort((a, b) => {
-        const aValue = a[sortConfig.key];
-        const bValue = b[sortConfig.key];
-
-        if (typeof aValue === "string" && typeof bValue === "string") {
-          return sortConfig.direction === "ascending"
-            ? aValue.localeCompare(bValue)
-            : bValue.localeCompare(aValue);
-        } else if (typeof aValue === "number" && typeof bValue === "number") {
-          return sortConfig.direction === "ascending"
-            ? aValue - bValue
-            : bValue - aValue;
-        } else {
-          return 0;
-        }
-      });
-    }
-    return sortableData;
+  const flattenDataFromBenchmarks = (): FlattenedModel[] =>
+    benchmarkData.map((b) => ({
+      provider: b.provider ?? "Unknown",
+      uri: b.source, // placeholder or use an optional `b.link` if available
+      name: b.model,
+      inputPrice: b.inputPrice,
+      outputPrice: b.outputPrice,
+      benchmark: b.benchmark ?? {},
+    }));
+
+
+
+  const filteredData = useMemo(() => {
+    if (!selectedProviders.length && !selectedModels.length) return data;
+
+    return data
+      .filter((p) => !selectedProviders.length || selectedProviders.includes(p.provider))
+      .map((p) => ({
+        ...p,
+        models: p.models.filter((m) => {
+          if (!selectedModels.length) return selectedProviders.includes(p.provider);
+          if (!selectedModels.length) return !selectedProviders.length || selectedProviders.includes(p.provider);
+          return selectedModels.includes(m.name);
+        }),
+      }))
+      .filter((p) => p.models.length > 0);
+  }, [data, selectedProviders, selectedModels]);
+
+  const benchmarkedModels = useMemo(() => {
+    return flattenDataFromBenchmarks();
+  }, []);
+
+
+
+  const filteredBenchmarkedModels = useMemo(() => {
+    return benchmarkedModels.filter((model) => {
+      const providerMatch =
+        selectedBenchmarkProviders.length === 0 || selectedBenchmarkProviders.includes(model.provider);
+      const modelMatch =
+        selectedBenchmarkModels.length === 0 || selectedBenchmarkModels.includes(model.name);
+
+      // When not linking, allow filtering by either
+      if (selectedBenchmarkProviders.length > 0 && selectedBenchmarkModels.length > 0) {
+        return providerMatch || modelMatch;
+      }
+
+      return providerMatch && modelMatch; // this handles the case where one or both are empty
+    });
+  }, [
+    benchmarkedModels,
+    selectedBenchmarkProviders,
+    selectedBenchmarkModels,
+
+  ]);
+
+  const sortedBenchmarkedModels = useMemo(() => {
+    if (!benchmarkSortConfig) return filteredBenchmarkedModels;
+
+    return [...filteredBenchmarkedModels].sort((a, b) => {
+      const key = benchmarkSortConfig.key;
+
+      const isTopLevelKey = ["provider", "name", "inputPrice", "outputPrice"].includes(key);
+
+      const aVal = isTopLevelKey
+        ? (a as any)[key]
+        : a.benchmark?.[key] ?? -Infinity;
+      const bVal = isTopLevelKey
+        ? (b as any)[key]
+        : b.benchmark?.[key] ?? -Infinity;
+
+      if (typeof aVal === "string" && typeof bVal === "string") {
+        return benchmarkSortConfig.direction === "ascending"
+          ? aVal.localeCompare(bVal)
+          : bVal.localeCompare(aVal);
+      }
+
+      return benchmarkSortConfig.direction === "ascending"
+        ? aVal - bVal
+        : bVal - aVal;
+    });
+  }, [filteredBenchmarkedModels, benchmarkSortConfig]);
+
+
+  const pricingProviders = useMemo(() => {
+    const grouped: Record<string, FlattenedModel[]> = {};
+
+    flattenDataFromPricing(data).forEach((model) => {
+      const key = model.provider;
+      if (!grouped[key]) grouped[key] = [];
+      grouped[key].push(model);
+    });
+
+    return Object.entries(grouped).map(([provider, models]) => ({
+      provider,
+      uri: models[0]?.uri ?? "#",
+      models: models.map(({ name, inputPrice, outputPrice }) => ({
+        name,
+        inputPrice,
+        outputPrice,
+      })),
+    }));
+  }, [data]);
+
+
+  const benchmarkProviders = useMemo(() => {
+    const grouped: Record<string, FlattenedModel[]> = {};
+
+    benchmarkedModels.forEach((model) => {
+      const key = model.provider;
+      if (!grouped[key]) grouped[key] = [];
+      grouped[key].push(model);
+    });
+
+    return Object.entries(grouped).map(([provider, models]) => ({
+      provider,
+      uri: models[0]?.uri ?? "#",
+      models: models.map(({ name, inputPrice, outputPrice }) => ({
+        name,
+        inputPrice,
+        outputPrice,
+      })),
+    }));
+  }, [benchmarkedModels]);
+
+
+
+
+
+
+  const sortedFlattenedData = useMemo(() => {
+    const flattened = flattenDataFromPricing(filteredData);
+    if (!sortConfig) return flattened;
+
+    return [...flattened].sort((a, b) => {
+      const aVal = a[sortConfig.key];
+      const bVal = b[sortConfig.key];
+
+      if (typeof aVal === "string" && typeof bVal === "string") {
+        return sortConfig.direction === "ascending" ? aVal.localeCompare(bVal) : bVal.localeCompare(aVal);
+      } else if (typeof aVal === "number" && typeof bVal === "number") {
+        return sortConfig.direction === "ascending" ? aVal - bVal : bVal - aVal;
+      }
+      return 0;
+    });
   }, [filteredData, sortConfig]);
 
   const requestSort = (key: keyof FlattenedModel) => {
-    let direction = "ascending";
-    if (
-      sortConfig &&
-      sortConfig.key === key &&
-      sortConfig.direction === "ascending"
-    ) {
-      direction = "descending";
-    }
+    const direction = sortConfig?.key === key && sortConfig.direction === "ascending" ? "descending" : "ascending";
     setSortConfig({ key, direction });
   };
 
   const toggleProviderExpansion = (provider: string) => {
     setExpandedProviders((prev) =>
-      prev.includes(provider)
-        ? prev.filter((p) => p !== provider)
-        : [...prev, provider]
+      prev.includes(provider) ? prev.filter((p) => p !== provider) : [...prev, provider]
     );
   };
 
-  const getModelsForSelectedProviders = () => {
-    if (!linkProviderModel) {
-      return data
-        .flatMap((provider) =>
-          provider.models.map((model) => ({
-            label: model.name,
-            value: model.name,
-            provider: provider.provider,
-          }))
-        )
-        .reduce(
-          (
-            acc: { label: string; value: string; provider: string }[],
-            curr: { label: string; value: string; provider: string }
-          ) => {
-            if (!acc.find((m) => m.value === curr.value)) {
-              acc.push(curr);
-            }
-            return acc;
-          },
-          []
-        );
-    }
 
-    return data
-      .filter((provider) => selectedProviders.includes(provider.provider))
-      .flatMap((provider) =>
-        provider.models.map((model) => ({
-          label: model.name,
-          value: model.name,
-          provider: provider.provider,
-        }))
-      )
-      .reduce(
-        (
-          acc: { label: string; value: string; provider: string }[],
-          curr: { label: string; value: string; provider: string }
-        ) => {
-          if (!acc.find((m) => m.value === curr.value)) {
-            acc.push(curr);
-          }
-          return acc;
-        },
-        []
-      );
-  };
 
   return (
     <Card className="w-full max-w-6xl mx-auto">
@@ -219,121 +235,46 @@ const App: React.FC = () => {
         <CardTitle>LLM Pricing Calculator</CardTitle>
       </CardHeader>
       <CardContent>
-        <div className="mb-4">
-          <p className="italic text-sm text-muted-foreground mb-4">
-            <a
-              href="https://huggingface.co/spaces/philschmid/llm-pricing"
-              className="underline"
-            >
-              This is a fork of philschmid tool: philschmid/llm-pricing
-            </a>
-          </p>
-          <h3 className="text-lg font-semibold mb-2">
-            Select Comparison Models
-          </h3>
-          <div className="grid grid-cols-2 md:grid-cols-3 lg:grid-cols-4 gap-4">
-            {data.map((provider) => (
-              <Collapsible
-                key={provider.provider}
-                open={expandedProviders.includes(provider.provider)}
-                onOpenChange={() => toggleProviderExpansion(provider.provider)}
-              >
-                <CollapsibleTrigger asChild>
-                  <Button variant="outline" className="w-full justify-between">
-                    {provider.provider}
-                    {expandedProviders.includes(provider.provider) ? (
-                      <ChevronDown className="h-4 w-4" />
-                    ) : (
-                      <ChevronRight className="h-4 w-4" />
-                    )}
-                  </Button>
-                </CollapsibleTrigger>
-                <CollapsibleContent className="mt-2">
-                  {provider.models.map((model) => (
-                    <div
-                      key={`${provider.provider}:${model.name}`}
-                      className="flex items-center space-x-2 mb-1"
-                    >
-                      <Checkbox
-                        id={`${provider.provider}:${model.name}`}
-                        checked={comparisonModels.includes(
-                          `${provider.provider}:${model.name}`
-                        )}
-                        onCheckedChange={(checked) => {
-                          if (checked) {
-                            setComparisonModels((prev) => [
-                              ...prev,
-                              `${provider.provider}:${model.name}`,
-                            ]);
-                          } else {
-                            setComparisonModels((prev) =>
-                              prev.filter(
-                                (m) =>
-                                  m !== `${provider.provider}:${model.name}`
-                              )
-                            );
-                          }
-                        }}
-                      />
-                      <label
-                        htmlFor={`${provider.provider}:${model.name}`}
-                        className="text-sm font-medium text-gray-700"
-                      >
-                        {model.name}
-                      </label>
-                    </div>
-                  ))}
-                </CollapsibleContent>
-              </Collapsible>
-            ))}
-          </div>
-        </div>
+        {/* Source Link */}
+        <p className="italic text-sm text-muted-foreground mb-4">
+          <a
+            href="https://huggingface.co/spaces/philschmid/llm-pricing"
+            className="underline"
+          >
+            This is a fork of philschmid tool: philschmid/llm-pricing
+          </a>
+        </p>
 
-        <div className="flex gap-4 mb-4">
+        {/* Comparison Model Selector */}
+        <h3 className="text-lg font-semibold mb-2">Select Comparison Models</h3>
+        <ComparisonSelector
+          data={data}
+          expanded={expandedProviders}
+          comparisonModels={comparisonModels}
+          onToggleExpand={toggleProviderExpansion}
+          onChangeModel={(modelId, checked) =>
+            setComparisonModels((prev) =>
+              checked ? [...prev, modelId] : prev.filter((m) => m !== modelId)
+            )
+          }
+        />
+
+        {/* Token Inputs */}
+        <div className="flex gap-4 mt-6 mb-4">
           <div className="flex-1">
-            <label
-              htmlFor="inputTokens"
-              className="block text-sm font-medium text-gray-700"
-            >
-              Input Tokens ({tokenCalculation})
-            </label>
-            <Input
-              id="inputTokens"
-              type="number"
-              value={inputTokens}
-              min={1}
-              onChange={(e) => setInputTokens(Number(e.target.value))}
-              className="mt-1"
-            />
+            <label className="block text-sm font-medium">Input Tokens ({tokenCalculation})</label>
+            <Input type="number" value={inputTokens} min={1} onChange={(e) => setInputTokens(Number(e.target.value))} />
           </div>
           <div className="flex-1">
-            <label
-              htmlFor="outputTokens"
-              className="block text-sm font-medium text-gray-700"
-            >
-              Output Tokens ({tokenCalculation})
-            </label>
-            <Input
-              id="outputTokens"
-              type="number"
-              value={outputTokens}
-              min={1}
-              onChange={(e) => setOutputTokens(Number(e.target.value))}
-              className="mt-1"
-            />
+            <label className="block text-sm font-medium">Output Tokens ({tokenCalculation})</label>
+            <Input type="number" value={outputTokens} min={1} onChange={(e) => setOutputTokens(Number(e.target.value))} />
           </div>
           <div className="flex-1">
-            <label
-              htmlFor="tokenCalculation"
-              className="block text-sm font-medium text-gray-700"
-            >
-              Token Calculation
-            </label>
+            <label className="block text-sm font-medium">Token Calculation</label>
             <select
-              id="tokenCalculation"
               value={tokenCalculation}
               onChange={(e) => setTokenCalculation(e.target.value)}
-              className="mt-1 block w-full pl-3 pr-10 py-2 text-base bg-white border focus:outline-none focus:ring-indigo-500  sm:text-sm rounded-md"
+              className="mt-1 block w-full pl-3 pr-10 py-2 text-base border rounded-md"
             >
               <option value="billion">Billion Tokens</option>
               <option value="million">Million Tokens</option>
@@ -343,196 +284,58 @@ const App: React.FC = () => {
           </div>
         </div>
 
-        <p className="italic text-sm text-muted-foreground mb-4">
-          Note: If you use Amazon Bedrock or Azure prices for Anthropic, Cohere
-          or OpenAI should be the same.
-        </p>
-        <div className="flex items-center space-x-2 mb-4">
-          <Switch
-            id="linkProviderModel"
-            checked={linkProviderModel}
-            onCheckedChange={setLinkProviderModel}
-          />
-          <label htmlFor="linkProviderModel" className="text-sm">
-            Link Provider and Model
-          </label>
-        </div>
+        {/* Pricing Table */}
+        <h2 className="text-lg font-semibold mb-2">Pricing Table</h2>
+        <PricingTable
+          data={sortedFlattenedData}
+          providers={pricingProviders}
+          selectedProviders={selectedProviders}
+          selectedModels={selectedModels}
+          onProviderChange={setSelectedProviders}
+          onModelChange={setSelectedModels}
+          comparisonModels={comparisonModels}
+          inputTokens={inputTokens}
+          outputTokens={outputTokens}
+          tokenCalculation={tokenCalculation}
+          requestSort={requestSort}
+          sortConfig={sortConfig}
+        />
+
+
+        {/* Benchmark Table */}
+        <h3 className="text-lg font-semibold mt-12 mb-2">Select Benchmark Metrics to Compare</h3>
+        <BenchmarkComparisonSelector
+          allMetrics={benchmarkMetricOrder.filter(
+            (metric) => benchmarkedModels.some((m) => m.benchmark?.[metric] !== undefined)
+          )}
+          selected={benchmarkComparisonMetrics}
+          onChange={(metric, checked) =>
+            setBenchmarkComparisonMetrics((prev) =>
+              checked ? [...prev, metric] : prev.filter((m) => m !== metric)
+            )
+          }
+        />
+
+        <h2 className="text-lg font-semibold mt-12 mb-2">Benchmark Table</h2>
+        <BenchmarkTable
+          data={sortedBenchmarkedModels}
+          comparisonMetrics={benchmarkComparisonMetrics}
+          requestSort={(key) => {
+            setBenchmarkSortConfig((prev) =>
+              prev?.key === key
+                ? { key, direction: prev.direction === "ascending" ? "descending" : "ascending" }
+                : { key, direction: "descending" }
+            );
+          }}
+          sortConfig={benchmarkSortConfig}
+          allProviders={benchmarkProviders}
+          selectedProviders={selectedBenchmarkProviders}
+          selectedModels={selectedBenchmarkModels}
+          onProviderChange={setSelectedBenchmarkProviders}
+          onModelChange={setSelectedBenchmarkModels}
+        />
+
 
-        <Table>
-          <TableHeader>
-            <TableRow>
-              <TableHead>
-                <button type="button" onClick={() => requestSort("provider")}>
-                  Provider{" "}
-                  {sortConfig?.key === "provider"
-                    ? sortConfig.direction === "ascending"
-                      ? "▲"
-                      : "▼"
-                    : null}
-                </button>
-              </TableHead>
-              <TableHead>
-                <button type="button" onClick={() => requestSort("name")}>
-                  Model{" "}
-                  {sortConfig?.key === "name"
-                    ? sortConfig.direction === "ascending"
-                      ? "▲"
-                      : "▼"
-                    : null}
-                </button>
-              </TableHead>
-
-              <TableHead>
-                <button type="button" onClick={() => requestSort("inputPrice")}>
-                  Input Price (million tokens)
-                  {sortConfig?.key === "inputPrice"
-                    ? sortConfig.direction === "ascending"
-                      ? "▲"
-                      : "▼"
-                    : null}
-                </button>
-              </TableHead>
-              <TableHead>
-                <button
-                  type="button"
-                  onClick={() => requestSort("outputPrice")}
-                >
-                  Output Price (million tokens)
-                  {sortConfig?.key === "outputPrice"
-                    ? sortConfig.direction === "ascending"
-                      ? "▲"
-                      : "▼"
-                    : null}
-                </button>
-              </TableHead>
-
-              <TableHead>
-                Total Price (per {tokenCalculation} tokens){" "}
-              </TableHead>
-              {comparisonModels.map((model) => (
-                <TableHead key={model} colSpan={2}>
-                  Compared to {model}
-                </TableHead>
-              ))}
-            </TableRow>
-            <TableRow>
-              <TableHead>
-                <MultiSelect
-                  options={
-                    data.map((provider) => ({
-                      label: provider.provider,
-                      value: provider.provider,
-                    })) || []
-                  }
-                  onValueChange={setSelectedProviders}
-                  defaultValue={selectedProviders}
-                />
-              </TableHead>
-              <TableHead>
-                <MultiSelect
-                  options={getModelsForSelectedProviders()}
-                  defaultValue={selectedModels}
-                  onValueChange={setSelectedModels}
-                />
-              </TableHead>
-              <TableHead />
-              <TableHead />
-              <TableHead />
-              {comparisonModels.flatMap((model) => [
-                <TableHead key={`${model}-input`}>Input</TableHead>,
-                <TableHead key={`${model}-output`}>Output</TableHead>,
-              ])}
-            </TableRow>
-          </TableHeader>
-          <TableBody>
-            {sortedFlattenedData.map((item) => (
-              <TableRow key={`${item.provider}-${item.name}`}>
-                <TableCell>
-                  {" "}
-                  <a href={item.uri} className="underline">
-                    {item.provider}
-                  </a>
-                </TableCell>
-                <TableCell>{item.name}</TableCell>
-
-                <TableCell>{item.inputPrice.toFixed(2)}</TableCell>
-                <TableCell>{item.outputPrice.toFixed(2)}</TableCell>
-
-                <TableCell className="font-bold">
-                  $
-                  {(
-                    calculatePrice(item.inputPrice, inputTokens) +
-                    calculatePrice(item.outputPrice, outputTokens)
-                  ).toFixed(2)}
-                </TableCell>
-
-                {comparisonModels.flatMap((comparisonModel) => {
-                  const [comparisonProvider, comparisonModelName] =
-                    comparisonModel.split(":");
-                  const comparisonModelData = data
-                    .find((p) => p.provider === comparisonProvider)
-                    ?.models.find((m) => m.name === comparisonModelName)!;
-                  return [
-                    <TableCell
-                      key={`${comparisonModel}-input`}
-                      className={`${
-                        parseFloat(
-                          calculateComparison(
-                            item.inputPrice,
-                            comparisonModelData.inputPrice
-                          )
-                        ) < 0
-                          ? "bg-green-100"
-                          : parseFloat(
-                              calculateComparison(
-                                item.inputPrice,
-                                comparisonModelData.inputPrice
-                              )
-                            ) > 0
-                          ? "bg-red-100"
-                          : ""
-                      }`}
-                    >
-                      {`${item.provider}:${item.name}` === comparisonModel
-                        ? "0.00%"
-                        : `${calculateComparison(
-                            item.inputPrice,
-                            comparisonModelData.inputPrice
-                          )}%`}
-                    </TableCell>,
-                    <TableCell
-                      key={`${comparisonModel}-output`}
-                      className={`${
-                        parseFloat(
-                          calculateComparison(
-                            item.outputPrice,
-                            comparisonModelData.outputPrice
-                          )
-                        ) < 0
-                          ? "bg-green-100"
-                          : parseFloat(
-                              calculateComparison(
-                                item.outputPrice,
-                                comparisonModelData.outputPrice
-                              )
-                            ) > 0
-                          ? "bg-red-100"
-                          : ""
-                      }`}
-                    >
-                      {`${item.provider}:${item.name}` === comparisonModel
-                        ? "0.00%"
-                        : `${calculateComparison(
-                            item.outputPrice,
-                            comparisonModelData.outputPrice
-                          )}%`}
-                    </TableCell>,
-                  ];
-                })}
-              </TableRow>
-            ))}
-          </TableBody>
-        </Table>
       </CardContent>
     </Card>
   );

src/components/BenchmarkComparisonSelector.tsx

@@ -0,0 +1,32 @@
+// components/BenchmarkComparisonSelector.tsx
+import React from "react";
+import { Checkbox } from "@/components/ui/checkbox";
+
+interface Props {
+  allMetrics: string[];
+  selected: string[];
+  onChange: (metric: string, checked: boolean) => void;
+}
+
+export const BenchmarkComparisonSelector: React.FC<Props> = ({
+  allMetrics,
+  selected,
+  onChange,
+}) => {
+  return (
+    <div className="grid grid-cols-2 sm:grid-cols-3 md:grid-cols-4 gap-2 mb-4">
+      {allMetrics.map((metric) => (
+        <div key={metric} className="flex items-center space-x-2">
+          <Checkbox
+            id={metric}
+            checked={selected.includes(metric)}
+            onCheckedChange={(checked) => onChange(metric, !!checked)}
+          />
+          <label htmlFor={metric} className="text-sm">
+            {metric.replace(/_/g, " ").toUpperCase()}
+          </label>
+        </div>
+      ))}
+    </div>
+  );
+};

src/components/BenchmarkTable.tsx

@@ -0,0 +1,173 @@
+import * as React from "react";
+import {
+    Table,
+    TableHeader,
+    TableBody,
+    TableRow,
+    TableHead,
+    TableCell,
+} from "@/components/ui/table";
+import { MultiSelect } from "@/components/ui/multi-select";
+import { Provider, FlattenedModel } from "@/App";
+
+interface BenchmarkTableProps {
+    data: FlattenedModel[];
+    comparisonMetrics?: string[];
+    requestSort: (key: string) => void;
+    sortConfig: {
+        key: string;
+        direction: "ascending" | "descending";
+    } | null;
+    allProviders: Provider[];
+    selectedProviders: string[];
+    selectedModels: string[];
+    onProviderChange: (values: string[]) => void;
+    onModelChange: (values: string[]) => void;
+}
+
+
+export const BenchmarkTable: React.FC<BenchmarkTableProps> = ({
+    data,
+    comparisonMetrics,
+    requestSort,
+    sortConfig,
+    allProviders,
+    selectedProviders,
+    selectedModels,
+    onProviderChange,
+    onModelChange,
+
+}) => {
+    const benchmarkMetrics = React.useMemo(() => {
+        if (!comparisonMetrics || comparisonMetrics.length === 0) return [];
+        return comparisonMetrics;
+    }, [comparisonMetrics]);
+
+    const getCellStyle = (value?: number) => {
+        if (value === undefined) return "";
+        if (value >= 85) return "bg-green-100";
+        if (value >= 60) return "bg-yellow-100";
+        return "bg-red-100";
+    };
+
+    const renderSortIndicator = (key: string) => {
+        if (sortConfig?.key !== key) return null;
+        return sortConfig.direction === "ascending" ? " ▲" : " ▼";
+    };
+
+    const modelOptions = React.useMemo(() => {
+        const flat = allProviders.flatMap((provider) =>
+            provider.models.map((model) => ({
+                label: model.name,
+                value: model.name,
+                provider: provider.provider,
+            }))
+        );
+
+const filtered = flat.filter((m) =>
+  !selectedProviders.length || selectedProviders.includes(m.provider)
+);
+
+
+        return Array.from(new Map(filtered.map((m) => [m.value, m])).values());
+    }, [allProviders, selectedProviders]);
+
+
+    return (
+        <>
+            <div className="flex items-center space-x-4 mb-4">
+                <div className="flex-1">
+                    <label className="text-sm font-medium block mb-1">Providers</label>
+                    <MultiSelect
+                        options={allProviders.map((p) => ({ label: p.provider, value: p.provider }))}
+                        defaultValue={selectedProviders}
+                        onValueChange={onProviderChange}
+                    />
+                </div>
+                <div className="flex-1">
+                    <label className="text-sm font-medium block mb-1">Models</label>
+                    <MultiSelect
+                        options={modelOptions.map(({ label, value }) => ({ label, value }))}
+                        defaultValue={selectedModels}
+                        onValueChange={onModelChange}
+                    />
+                </div>
+
+            </div>
+
+            <Table>
+                <TableHeader>
+                    <TableRow>
+                        <TableHead>
+                            <button
+                                onClick={() => requestSort("provider")}
+                                className="font-medium underline underline-offset-2"
+                            >
+                                Provider{renderSortIndicator("provider")}
+                            </button>
+                        </TableHead>
+                        <TableHead>
+                            <button
+                                onClick={() => requestSort("name")}
+                                className="font-medium underline underline-offset-2"
+                            >
+                                Model{renderSortIndicator("name")}
+                            </button>
+                        </TableHead>
+                        <TableHead>
+                            <button
+                                onClick={() => requestSort("inputPrice")}
+                                className="font-medium underline underline-offset-2"
+                            >
+                                Input Price (USD){renderSortIndicator("inputPrice")}
+                            </button>
+                        </TableHead>
+                        <TableHead>
+                            <button
+                                onClick={() => requestSort("outputPrice")}
+                                className="font-medium underline underline-offset-2"
+                            >
+                                Output Price (USD){renderSortIndicator("outputPrice")}
+                            </button>
+                        </TableHead>
+
+                        {benchmarkMetrics.map((metric) => (
+                            <TableHead key={metric}>
+                                <button
+                                    onClick={() => requestSort(metric)}
+                                    className="font-medium underline underline-offset-2"
+                                >
+                                    {metric.replace(/_/g, " ").toUpperCase()}
+                                    {renderSortIndicator(metric)}
+                                </button>
+                            </TableHead>
+                        ))}
+                    </TableRow>
+                </TableHeader>
+                <TableBody>
+                    {data.map((model) => (
+                        <TableRow key={`${model.provider}-${model.name}`}>
+                            <TableCell>
+                                <a href={model.uri} className="underline">
+                                    {model.provider}
+                                </a>
+                            </TableCell>
+                            <TableCell>{model.name}</TableCell>
+                            <TableCell>${model.inputPrice.toFixed(2)}</TableCell>
+                            <TableCell>${model.outputPrice.toFixed(2)}</TableCell>
+
+                            {benchmarkMetrics.map((metric) => {
+                                const score = model.benchmark?.[metric];
+                                return (
+                                    <TableCell key={metric} className={getCellStyle(score)}>
+                                        {score !== undefined ? `${score.toFixed(1)}%` : "-"}
+                                    </TableCell>
+                                );
+                            })}
+                        </TableRow>
+                    ))}
+                </TableBody>
+            </Table>
+        </>
+    );
+};

src/components/ComparisonSelector.tsx

@@ -0,0 +1,64 @@
+import React from "react";
+import {
+  Collapsible,
+  CollapsibleContent,
+  CollapsibleTrigger,
+} from "@/components/ui/collapsible";
+import { Button } from "@/components/ui/button";
+import { Checkbox } from "@/components/ui/checkbox";
+import { ChevronDown, ChevronRight } from "lucide-react";
+import { Provider } from "@/App";
+
+interface Props {
+  data: Provider[];
+  expanded: string[];
+  comparisonModels: string[];
+  onToggleExpand: (provider: string) => void;
+  onChangeModel: (value: string, checked: boolean) => void;
+}
+
+export const ComparisonSelector: React.FC<Props> = ({
+  data,
+  expanded,
+  comparisonModels,
+  onToggleExpand,
+  onChangeModel,
+}) => (
+  <div className="grid grid-cols-2 md:grid-cols-3 lg:grid-cols-4 gap-4">
+    {data.map((provider) => (
+      <Collapsible
+        key={provider.provider}
+        open={expanded.includes(provider.provider)}
+        onOpenChange={() => onToggleExpand(provider.provider)}
+      >
+        <CollapsibleTrigger asChild>
+          <Button variant="outline" className="w-full justify-between">
+            {provider.provider}
+            {expanded.includes(provider.provider) ? (
+              <ChevronDown className="h-4 w-4" />
+            ) : (
+              <ChevronRight className="h-4 w-4" />
+            )}
+          </Button>
+        </CollapsibleTrigger>
+        <CollapsibleContent className="mt-2">
+          {provider.models.map((model) => {
+            const id = `${provider.provider}:${model.name}`;
+            return (
+              <div key={id} className="flex items-center space-x-2 mb-1">
+                <Checkbox
+                  id={id}
+                  checked={comparisonModels.includes(id)}
+                  onCheckedChange={(checked) => onChangeModel(id, !!checked)}
+                />
+                <label htmlFor={id} className="text-sm font-medium text-gray-700">
+                  {model.name}
+                </label>
+              </div>
+            );
+          })}
+        </CollapsibleContent>
+      </Collapsible>
+    ))}
+  </div>
+);

src/components/PricingTable.tsx

@@ -0,0 +1,175 @@
+import * as React from "react";
+import {
+  Table,
+  TableHeader,
+  TableBody,
+  TableRow,
+  TableHead,
+  TableCell,
+} from "@/components/ui/table";
+import { MultiSelect } from "@/components/ui/multi-select";
+import { FlattenedModel, Provider } from "@/App";
+
+interface PricingTableProps {
+  data: FlattenedModel[];
+  providers: Provider[];
+  selectedProviders: string[];
+  selectedModels: string[];
+  onProviderChange: (values: string[]) => void;
+  onModelChange: (values: string[]) => void;
+  comparisonModels: string[];
+  inputTokens: number;
+  outputTokens: number;
+  tokenCalculation: string;
+  requestSort: (key: keyof FlattenedModel) => void;
+  sortConfig: {
+    key: keyof FlattenedModel;
+    direction: string;
+  } | null;
+
+}
+
+export const PricingTable: React.FC<PricingTableProps> = ({
+  data,
+  providers,
+  selectedProviders,
+  selectedModels,
+  onProviderChange,
+  onModelChange,
+  comparisonModels,
+  inputTokens,
+  outputTokens,
+  tokenCalculation,
+  requestSort,
+  sortConfig,
+
+}) => {
+  const calculatePrice = (price: number, tokens: number): number => {
+    let multiplier = 1;
+    if (tokenCalculation === "thousand") multiplier = 1e-3;
+    else if (tokenCalculation === "unit") multiplier = 1e-6;
+    else if (tokenCalculation === "billion") multiplier = 1e3;
+    return price * tokens * multiplier;
+  };
+
+  const calculateComparison = (modelPrice: number, comparisonPrice: number): string => {
+    if (comparisonPrice === 0) return "∞";
+    return (((modelPrice - comparisonPrice) / comparisonPrice) * 100).toFixed(2);
+  };
+
+  const getModelsForSelectedProviders = () => {
+    const flatModels = providers.flatMap((provider) =>
+      provider.models.map((model) => ({
+        label: model.name,
+        value: model.name,
+        provider: provider.provider,
+      }))
+    );
+
+const filtered = flatModels.filter((m) =>
+  !selectedProviders.length || selectedProviders.includes(m.provider)
+);
+
+
+    // Remove duplicates
+    return Array.from(new Map(filtered.map((m) => [m.value, m])).values());
+  };
+
+  return (
+    <Table>
+      <TableHeader>
+        <TableRow>
+          <TableHead>
+            <button onClick={() => requestSort("provider")}>
+              Provider {sortConfig?.key === "provider" ? (sortConfig.direction === "ascending" ? "▲" : "▼") : null}
+            </button>
+          </TableHead>
+          <TableHead>
+            <button onClick={() => requestSort("name")}>
+              Model {sortConfig?.key === "name" ? (sortConfig.direction === "ascending" ? "▲" : "▼") : null}
+            </button>
+          </TableHead>
+          <TableHead>
+            <button onClick={() => requestSort("inputPrice")}>
+              Input Price (million tokens) {sortConfig?.key === "inputPrice" ? (sortConfig.direction === "ascending" ? "▲" : "▼") : null}
+            </button>
+          </TableHead>
+          <TableHead>
+            <button onClick={() => requestSort("outputPrice")}>
+              Output Price (million tokens) {sortConfig?.key === "outputPrice" ? (sortConfig.direction === "ascending" ? "▲" : "▼") : null}
+            </button>
+          </TableHead>
+          <TableHead>Total Price (per {tokenCalculation} tokens)</TableHead>
+          {comparisonModels.map((model) => (
+            <TableHead key={model} colSpan={2}>
+              Compared to {model}
+            </TableHead>
+          ))}
+        </TableRow>
+        <TableRow>
+          <TableHead>
+            <MultiSelect
+              options={providers.map((p) => ({ label: p.provider, value: p.provider }))}
+              defaultValue={selectedProviders}
+              onValueChange={onProviderChange}
+            />
+          </TableHead>
+          <TableHead>
+            <MultiSelect
+              options={getModelsForSelectedProviders()}
+              defaultValue={selectedModels}
+              onValueChange={onModelChange}
+            />
+          </TableHead>
+          <TableHead />
+          <TableHead />
+          <TableHead />
+          {comparisonModels.flatMap((model) => [
+            <TableHead key={`${model}-input`}>Input</TableHead>,
+            <TableHead key={`${model}-output`}>Output</TableHead>,
+          ])}
+        </TableRow>
+      </TableHeader>
+      <TableBody>
+        {data.map((item) => (
+          <TableRow key={`${item.provider}-${item.name}`}>
+            <TableCell>
+              <a href={item.uri} className="underline">
+                {item.provider}
+              </a>
+            </TableCell>
+            <TableCell>{item.name}</TableCell>
+            <TableCell>{item.inputPrice.toFixed(2)}</TableCell>
+            <TableCell>{item.outputPrice.toFixed(2)}</TableCell>
+            <TableCell className="font-bold">
+              ${(
+                calculatePrice(item.inputPrice, inputTokens) +
+                calculatePrice(item.outputPrice, outputTokens)
+              ).toFixed(2)}
+            </TableCell>
+            {comparisonModels.flatMap((model) => {
+              const [comparisonProvider, comparisonModelName] = model.split(":");
+              const comparisonModel = providers
+                .find((p) => p.provider === comparisonProvider)
+                ?.models.find((m) => m.name === comparisonModelName);
+
+              if (!comparisonModel) return [<TableCell key="missing">–</TableCell>, <TableCell key="missing2">–</TableCell>];
+
+              const inputDelta = calculateComparison(item.inputPrice, comparisonModel.inputPrice);
+              const outputDelta = calculateComparison(item.outputPrice, comparisonModel.outputPrice);
+
+              return [
+                <TableCell key={`${model}-input`} className={parseFloat(inputDelta) < 0 ? "bg-green-100" : parseFloat(inputDelta) > 0 ? "bg-red-100" : ""}>
+                  {`${item.provider}:${item.name}` === model ? "0.00%" : `${inputDelta}%`}
+                </TableCell>,
+                <TableCell key={`${model}-output`} className={parseFloat(outputDelta) < 0 ? "bg-green-100" : parseFloat(outputDelta) > 0 ? "bg-red-100" : ""}>
+                  {`${item.provider}:${item.name}` === model ? "0.00%" : `${outputDelta}%`}
+                </TableCell>,
+              ];
+            })}
+          </TableRow>
+        ))}
+      </TableBody>
+    </Table>
+  );
+};

src/lib/benchmarks/ index.ts

@@ -0,0 +1,11 @@
+import { Benchmark } from "./types";
+import { xaiBenchmarks } from "./xai";
+import { googleBenchmarks } from "./google";
+// import other sources here as you add them
+// import { openaiBenchmarks } from "./openai";
+
+export const benchmarkData: Benchmark[] = [
+  ...xaiBenchmarks,
+  ...googleBenchmarks,
+  // ...openaiBenchmarks,
+];

src/lib/benchmarks/google.ts

@@ -0,0 +1,153 @@
+import { Benchmark } from "./types";
+
+
+export const googleBenchmarks: Benchmark[] = [
+    {
+        model: "Gemini Diffusion",
+        provider: "Google",
+        inputPrice: 0,
+        outputPrice: 0,
+        benchmark: {
+            livecodebench_v6: 30.9,
+            bigcodebench: 45.4,
+            lbpp_v2: 56.8,
+            swe_bench_verified: 22.9,
+            humaneval: 89.6,
+            mbpp: 76.0,
+            gpqa_diamond: 40.4,
+            aime_2025: 23.3,
+            bigbench_extra_hard: 15.0,
+            global_mmlu_lite: 69.1,
+        },
+        source: "https://deepmind.google/models/gemini-diffusion/",
+    },
+    {
+        model: "Gemini 2.0 Flash-Lite",
+        provider: "Google",
+        inputPrice: 0.10,
+        outputPrice: 0.40,
+        benchmark: {
+            livecodebench_v6: 28.5,
+            bigcodebench: 45.8,
+            lbpp_v2: 56.0,
+            swe_bench_verified: 28.5,
+            humaneval: 90.2,
+            mbpp: 75.8,
+            gpqa_diamond: 56.5,
+            aime_2025: 20.0,
+            bigbench_extra_hard: 21.0,
+            global_mmlu_lite: 79.0,
+        },
+        source: "https://deepmind.google/models/gemini-diffusion/",
+    },
+
+    {
+        model: "Gemini 2.5 Flash Preview (05-20)",
+        provider: "Google",
+        inputPrice: 0.15,
+        outputPrice: 3.5,
+        source: "https://ai.google.dev/gemini-api/docs/thinking",
+        benchmark: {
+            aime_2025: 72.0,
+            gpqa_diamond: 82.8,
+            simpleqa: 26.9,
+            global_mmlu_lite: 88.4,
+            swe_bench_verified: 60.4,
+            livecodebench_v6: 63.9,
+            mmmu: 79.7,
+            lbpp_v2: 61.9,
+            bigcodebench: 56.7,
+            facts_grounding: 85.3,
+            humanitys_last_exam: 11.0,
+            mrcr_v2_avg_128k: 74.0,
+            mrcr_v2_pointwise_1m: 32.0,
+
+        },
+    },
+    {
+        model: "Gemini 2.5 Flash Preview (04-17) Thinking",
+        provider: "Google",
+        inputPrice: 0.15,
+        outputPrice: 3.5,
+        source: "https://ai.google.dev/gemini-api/docs/thinking",
+        benchmark: {
+            aime_2025: 78.0,
+            gpqa_diamond: 78.3,
+            simpleqa: 29.7,
+            global_mmlu_lite: 88.4,
+            livecodebench_v6: 63.5,
+            lbpp_v2: 51.1,
+            bigcodebench: 44.2,
+            mmmu: 76.7,
+            humanitys_last_exam: 12.1
+
+        },
+    },
+    {
+        model: "Gemini 2.0 Flash",
+        provider: "Google",
+        inputPrice: 0.1,
+        outputPrice: 0.4,
+        source: "https://ai.google.dev/gemini-api/docs/thinking",
+        benchmark: {
+            aime_2025: 27.5,
+            gpqa_diamond: 60.1,
+            simpleqa: 29.9,
+            global_mmlu_lite: 83.4,
+            livecodebench_v6: 34.5,
+            lbpp_v2: 22.2,
+            mmmu: 71.7,
+            facts_grounding: 84.6,
+            humanitys_last_exam: 5.1,
+            mrcr_v2_avg_128k: 36.0,
+            mrcr_v2_pointwise_1m: 6.0,
+
+        },
+    },
+    {
+        model: "Gemini 2.5 Pro Preview (05-06)",
+        provider: "Google",
+        inputPrice: 2.5,
+        outputPrice: 15.0,
+        source: "https://blog.google/products/gemini/gemini-2-5-pro-updates/",
+        benchmark: {
+            humanitys_last_exam: 17.8,
+            gpqa_diamond: 83.0,
+            aime_2025: 83.0,
+            livecodebench_v6: 75.6,
+            lbpp_v2: 76.5,
+            bigcodebench: 72.7,
+            swe_bench_verified: 63.2,
+            simpleqa: 50.8,
+            mmmu: 79.6,
+
+            video_mme: 84.8,
+            mrcr_v2_avg_128k: 93.0,
+            mrcr_v2_pointwise_1m: 82.9,
+            global_mmlu_lite: 88.6,
+        },
+    },
+    {
+        model: "Gemini 2.5 Pro Experimental (03-25)",
+        provider: "Google",
+        inputPrice: 2.5,
+        outputPrice: 15.0,
+        source: "https://blog.google/technology/google-deepmind/gemini-model-thinking-updates-march-2025/",
+        benchmark: {
+            humanitys_last_exam: 18.8,
+            gpqa_diamond: 84.0,
+            aime_2025: 86.7,
+            livecodebench_v6: 70.4,
+            lbpp_v2: 74.0,
+            bigcodebench: 68.6,
+            swe_bench_verified: 63.8,
+            simpleqa: 52.9,
+            mmmu: 81.7,
+            mrcr_v2_avg_128k: 94.5,
+            mrcr_v2_pointwise_1m: 83.1,
+            global_mmlu_lite: 89.8,
+        },
+    },
+
+
+];

src/lib/benchmarks/types.ts

@@ -0,0 +1,76 @@
+export type BenchmarkMetric =
+  // Most common and high-priority
+  | "simpleqa"
+  | "mmlu_pro"
+  | "gpqa"
+  | "egoschema"
+  | "loft"
+  | "mmmu"
+  | "lcb"
+  | "aime_24"
+  | "aime_2025"
+  | "gpqa_diamond"
+
+  // Code benchmarks (frequent)
+  | "humaneval"
+  | "mbpp"
+  | "bigcodebench"
+  | "livecodebench_v6"
+  | "swe_bench_verified"
+  | "lbpp_v2"
+
+  // General reasoning & robustness
+  | "bigbench_extra_hard"
+  | "global_mmlu_lite"
+
+  // Optional: less frequent but still potentially useful
+  | "facts_grounding"
+  | "humanitys_last_exam"
+  | "mrcr_v2_avg_128k"
+  | "mrcr_v2_pointwise_1m"
+  | "video_mme";
+
+// Note: "vibe_eval_reka" is intentionally excluded for now.
+
+export interface Benchmark {
+  model: string;
+  provider: string;
+  benchmark: Partial<Record<BenchmarkMetric, number>>;
+  inputPrice: number;
+  outputPrice: number;
+  source: string;
+  version?: string;
+}
+
+export const benchmarkMetricOrder: BenchmarkMetric[] = [
+  // Most common and high-priority
+  "simpleqa",
+  "mmlu_pro",
+  "gpqa",
+  "egoschema",
+  "loft",
+  "mmmu",
+  "lcb",
+  "aime_24",
+  "aime_2025",
+  "gpqa_diamond",
+
+  // // Code benchmarks (frequent)
+  // "humaneval",
+  // "mbpp",
+  // "bigcodebench",
+  // "livecodebench_v6",
+  // "swe_bench_verified",
+  // "lbpp_v2",
+
+  // // General reasoning & robustness
+  // "bigbench_extra_hard",
+  // "global_mmlu_lite",
+
+  // // Optional: less frequent but still potentially useful
+  // "facts_grounding",
+  // "humanitys_last_exam",
+  // "mrcr_v2_avg_128k",
+  // "mrcr_v2_pointwise_1m",
+  // "video_mme",
+];

src/lib/benchmarks/xai.ts

@@ -0,0 +1,39 @@
+import { Benchmark } from "./types";
+
+
+export const xaiBenchmarks: Benchmark[] = [
+    {
+        model: "Grok 3 Beta",
+        provider: "Xai",
+        inputPrice: 3.00,
+        outputPrice: 15.00,
+        benchmark: {
+            aime_24: 52.2,
+            gpqa: 75.4,
+            lcb: 57.0,
+            mmlu_pro: 79.9,
+            loft: 83.3,
+            simpleqa: 43.6,
+            mmmu: 73.2,
+            egoschema: 74.5,
+        },
+        source: "https://x.ai/news/grok-3",
+    },
+    {
+        model: "Grok 3 mini Beta",
+        provider: "Xai",
+        inputPrice: 0.30,
+        outputPrice: 0.50,
+        benchmark: {
+            aime_24: 39.7,
+            gpqa: 66.2,
+            lcb: 41.5,
+            mmlu_pro: 78.9,
+            loft: 83.1,
+            simpleqa: 21.7,
+            mmmu: 69.4,
+            egoschema: 74.3,
+        },
+        source: "https://x.ai/news/grok-3",
+    },
+];