fix highlighting

app.py

@@ -99,11 +99,18 @@ with gr.Blocks() as demo:
         table_feat = table_feat.sort_values("activation", ascending=False)
 
         texts = table_feat["token"].apply(
-            lambda x: tokenizer.decode(token_table[max(0, x - nearby - 1):x + nearby + 1]["tokens"].to_numpy())
-        )
+            lambda x: [tokenizer.decode(y).replace("\n", " ") for y in token_table[max(0, x - nearby + 1):x + nearby + 2]["tokens"].to_numpy()]
+        ).tolist()
 
-        texts = [tokenizer.tokenize(text) for text in texts]
+        # texts = [tokenizer.tokenize(text) for text in texts]
         activations = table_feat["nearby"].to_numpy()
+
+        activations = [a for i, a in enumerate(activations) if len(texts[i]) > 0]
+        texts = [text for text in texts if len(text) > 0]
+
+        for t, a in zip(texts, activations):
+            assert len(t) == len(a)
+
         if len(activations) > 0:
             activations = np.stack(activations) * stride
             max_act = table_feat["activation"].max()

weights/caches/phi-l20-r5-st0.25x128-activations.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ba0048dbd58dc56a887ed25f44b13a27e79e98739df0233e9ca6caaaa43f132f
-size 18612669
+oid sha256:31c0ae67e80ebf495021b96340773e0e8191b447f1cc8eb72e2e302e5aa4833c
+size 26258945