correcting kl_div parameters and new graph for winning perc

app.py

@@ -8,6 +8,7 @@ import logging
 from scripts.metrics import (
     compute_weekly_metrics_by_market_creator,
     compute_weekly_metrics_by_trader_type,
+    compute_winning_metrics_by_trader,
 )
 from tabs.trader_plots import (
     plot_trader_metrics_by_market_creator,
@@ -15,6 +16,7 @@ from tabs.trader_plots import (
     default_trader_metric,
     trader_metric_choices,
     get_metrics_text,
+    plot_winning_metric_per_trader,
 )
 
 from tabs.market_plots import plot_kl_div_per_market
@@ -112,6 +114,9 @@ print(weekly_metrics_by_market_creator.head())
 weekly_metrics_by_trader_type = compute_weekly_metrics_by_trader_type(
     trader_agents_data
 )
+weekly_winning_metrics = compute_winning_metrics_by_trader(
+    trader_agents_data=trader_agents_data
+)
 with demo:
     gr.HTML("<h1>Trader agents monitoring dashboard </h1>")
     gr.Markdown(
@@ -195,5 +200,10 @@ with demo:
                     kl_div_plot = plot_kl_div_per_market(closed_markets=closed_markets)
                 with gr.Column(scale=1):
                     metrics_text = get_metrics_text()
+        with gr.TabItem("🎖️Weekly winning trades % per trader"):
+            with gr.Row():
+                gr.Markdown("# Winning trades percentage from weekly trades by trader")
+            with gr.Row():
+                winning_metric = plot_winning_metric_per_trader(weekly_winning_metrics)
 
 demo.queue(default_concurrency_limit=40).launch()

data/closed_markets_div.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:01028e48165f8e468cd377da59e13da584a0938cdc64549dee2a1c523d6e1b13
-size 48695
+oid sha256:8d2583a54d7d4b38e51c2bf1e808a710e6a4b232009b5ae8917ce0534b045d94
+size 48882

notebooks/closed_markets.ipynb

@@ -1455,6 +1455,112 @@
    "source": [
     "len(markets_div)"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "import math"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def kl_divergence(p, q):\n",
+    "    \"\"\"\n",
+    "    Compute KL divergence for a single sample with two probabilities.\n",
+    "\n",
+    "    :param p: First probability (true distribution)\n",
+    "    :param q: Second probability (approximating distribution)\n",
+    "    :return: KL divergence value\n",
+    "    \"\"\"\n",
+    "    # Ensure probabilities sum to 1\n",
+    "    p = np.array([p, 1 - p])\n",
+    "    q = np.array([q, 1 - q])\n",
+    "\n",
+    "    # Avoid division by zero\n",
+    "    epsilon = 1e-10\n",
+    "    q = np.clip(q, epsilon, 1 - epsilon)\n",
+    "\n",
+    "    # Compute KL divergence\n",
+    "    kl_div = np.sum(p * np.log(p / q))\n",
+    "\n",
+    "    return kl_div"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "3.791663620863367"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "p = 0.8145\n",
+    "q = 1.0\n",
+    "kl_divergence(p,q)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0.16397451204513597"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "p = 0.99\n",
+    "q = 0.8145\n",
+    "kl_divergence(p, q)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "KL divergence: 0.16397451204513597\n"
+     ]
+    }
+   ],
+   "source": [
+    "from scipy.special import kl_div\n",
+    "\n",
+    "# For multiple probabilities\n",
+    "p = np.array([0.99, 0.01])\n",
+    "q = np.array([0.8145, 0.1855])\n",
+    "\n",
+    "kl = np.sum(kl_div(p, q))\n",
+    "print(f\"KL divergence: {kl}\")"
+   ]
   }
  ],
  "metadata": {

scripts/closed_markets_divergence.py

@@ -164,13 +164,13 @@ def market_KL_divergence(market_row: pd.DataFrame) -> float:
     """Function to compute the divergence based on the formula
     Formula in https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence"""
     current_answer = market_row.currentAnswer  # "yes", "no"
-    candidate_prob = market_row.first_outcome_prob
-    target_prob = 1.0  # for yes outcome
+    approx_prob = market_row.first_outcome_prob
+    true_prob = 0.99  # for yes outcome (99% is the max we can specify to avoid nan)
     if current_answer == "no":
-        target_prob = 0.0  # = 0% for yes outcome
+        true_prob = 0.01  # = 0.1% for yes outcome and 99% for no
 
     # we have only one sample, the final probability based on tokens
-    return kl_divergence(candidate_prob, target_prob)
+    return kl_divergence(true_prob, approx_prob)
 
 
 def compute_tokens_prob(token_amounts: list) -> list:

scripts/metrics.py

@@ -76,6 +76,28 @@ def compute_trader_metrics_by_market_creator(
     return metrics
 
 
+def compute_winning_metric_per_trader_per_market_creator(
+    trader_address: str, week_traders_data: pd.DataFrame, market_creator: str = "all"
+) -> float:
+    assert "market_creator" in week_traders_data.columns
+    filtered_traders_data = week_traders_data.loc[
+        week_traders_data["trader_address"] == trader_address
+    ]
+    if market_creator != "all":  # compute only for the specific market creator
+        filtered_traders_data = filtered_traders_data.loc[
+            filtered_traders_data["market_creator"] == market_creator
+        ]
+        if len(filtered_traders_data) == 0:
+            tqdm.write(f"No data. Skipping market creator {market_creator}")
+            return None  # No Data
+    winning_perc = (
+        filtered_traders_data["winning_trade"].sum()
+        / filtered_traders_data["winning_trade"].count()
+        * 100.0
+    )
+    return winning_perc
+
+
 def merge_trader_metrics(
     trader: str, weekly_data: pd.DataFrame, week: str
 ) -> pd.DataFrame:
@@ -143,6 +165,47 @@ def merge_trader_metrics_by_type(
     return result
 
 
+def merge_winning_metrics_by_trader(
+    trader: str, weekly_data: pd.DataFrame, week: str
+) -> pd.DataFrame:
+    trader_metrics = []
+    # computation as specification 1 for all market creators
+    winning_metrics_all = {}
+    winning_metric_all = compute_winning_metric_per_trader_per_market_creator(
+        trader, weekly_data, market_creator="all"
+    )
+    winning_metrics_all["winning_perc"] = winning_metric_all
+    winning_metrics_all["month_year_week"] = week
+    winning_metrics_all["market_creator"] = "all"
+    trader_metrics.append(winning_metrics_all)
+
+    # computation as specification 1 for quickstart markets
+    winning_metrics_qs = {}
+    winning_metric = compute_winning_metric_per_trader_per_market_creator(
+        trader, weekly_data, market_creator="quickstart"
+    )
+    if winning_metric:
+        winning_metrics_qs["winning_perc"] = winning_metric
+        winning_metrics_qs["month_year_week"] = week
+        winning_metrics_qs["market_creator"] = "quickstart"
+        trader_metrics.append(winning_metrics_qs)
+
+    # computation as specification 1 for pearl markets
+    winning_metrics_pearl = {}
+    winning_metric = compute_winning_metric_per_trader_per_market_creator(
+        trader, weekly_data, market_creator="pearl"
+    )
+    if winning_metric:
+        winning_metrics_pearl["winning_perc"] = winning_metric
+        winning_metrics_pearl["month_year_week"] = week
+        winning_metrics_pearl["market_creator"] = "pearl"
+        trader_metrics.append(winning_metrics_pearl)
+
+    result = pd.DataFrame.from_dict(trader_metrics, orient="columns")
+    # tqdm.write(f"Total length of all winning metrics for this week = {len(result)}")
+    return result
+
+
 def compute_weekly_metrics_by_market_creator(
     trader_agents_data: pd.DataFrame,
 ) -> pd.DataFrame:
@@ -179,3 +242,22 @@ def compute_weekly_metrics_by_trader_type(
             contents.append(merge_trader_metrics_by_type(trader, weekly_data, week))
     print("End computing all weekly metrics by trader types")
     return pd.concat(contents, ignore_index=True)
+
+
+def compute_winning_metrics_by_trader(
+    trader_agents_data: pd.DataFrame,
+) -> pd.DataFrame:
+    """Function to compute the winning metrics at the trader level per week and with different market creators"""
+    contents = []
+    all_weeks = list(trader_agents_data.month_year_week.unique())
+    for week in all_weeks:
+        weekly_data = trader_agents_data.loc[
+            trader_agents_data["month_year_week"] == week
+        ]
+        print(f"Computing weekly metrics for week ={week} by trader type")
+        # traverse each trader agent
+        traders = list(weekly_data.trader_address.unique())
+        for trader in tqdm(traders, desc=f"Trader' metrics", unit="metrics"):
+            contents.append(merge_winning_metrics_by_trader(trader, weekly_data, week))
+    print("End computing all weekly winning metrics by trader")
+    return pd.concat(contents, ignore_index=True)

tabs/trader_plots.py

@@ -114,3 +114,25 @@ def plot_trader_metrics_by_trader_type(metric_name: str, traders_df: pd.DataFram
     return gr.Plot(
         value=fig,
     )
+
+
+def plot_winning_metric_per_trader(traders_winning_df: pd.DataFrame) -> gr.Plot:
+    fig = px.box(
+        traders_winning_df,
+        x="month_year_week",
+        y="winning_perc",
+        color="market_creator",
+        color_discrete_sequence=["purple", "goldenrod", "darkgreen"],
+        category_orders={"market_creator": ["pearl", "quickstart", "all"]},
+    )
+    fig.update_traces(boxmean=True)
+    fig.update_layout(
+        xaxis_title="Week",
+        yaxis_title="Weekly winning percentage %",
+        legend=dict(yanchor="top", y=0.5),
+    )
+    fig.update_xaxes(tickformat="%b %d\n%Y")
+
+    return gr.Plot(
+        value=fig,
+    )