Fix

table.py

@@ -1,6 +1,7 @@
 import datasets
 import polars as pl
 from loguru import logger
+from polars import datatypes as pdt
 
 BASE_REPO_ID = "ai-conferences/ICLR2025"
 PATCH_REPO_ID = "ai-conferences/ICLR2025-patches"
@@ -8,13 +9,22 @@ PAPER_PAGE_REPO_ID = "hysts-bot-data/paper-pages-slim"
 
 
 def get_patch_latest_values(
-    df: pl.DataFrame, all_columns: list[str], id_col: str, timestamp_col: str = "timestamp"
+    df: pl.DataFrame, all_columns: list[str], id_col: str, timestamp_col: str = "timestamp", delimiter: str = ","
 ) -> pl.DataFrame:
     df = df.sort(timestamp_col)
-    update_columns = [col for col in df.columns if col not in (id_col, timestamp_col)]
 
-    melted = df.unpivot(on=update_columns, index=[timestamp_col, id_col])
-    melted = melted.drop_nulls()
+    list_cols = [
+        col for col, dtype in df.schema.items() if col not in (id_col, timestamp_col) and dtype.base_type() is pdt.List
+    ]
+    df = df.with_columns(
+        [
+            pl.when(pl.col(c).is_not_null()).then(pl.col(c).list.join(delimiter)).otherwise(None).alias(c)
+            for c in list_cols
+        ]
+    )
+
+    update_columns = [col for col in df.columns if col not in (id_col, timestamp_col)]
+    melted = df.unpivot(on=update_columns, index=[timestamp_col, id_col]).drop_nulls()
 
     latest_rows = (
         melted.sort(timestamp_col)
@@ -23,9 +33,16 @@ def get_patch_latest_values(
         .pivot("variable", index=id_col, values="value")
     )
 
-    for col in all_columns:
-        if col != "id" and col not in latest_rows.columns:
-            latest_rows = latest_rows.with_columns(pl.lit(None).alias(col))
+    latest_rows = latest_rows.with_columns(
+        [
+            pl.when(pl.col(c).is_not_null()).then(pl.col(c).str.split(delimiter)).otherwise(None).alias(c)
+            for c in list_cols
+        ]
+    )
+
+    missing_cols = [c for c in all_columns if c not in latest_rows.columns and c != id_col]
+    if missing_cols:
+        latest_rows = latest_rows.with_columns([pl.lit(None).alias(c) for c in missing_cols])
 
     return latest_rows.select([id_col] + [col for col in all_columns if col != id_col])