Update app.py
Browse files
app.py
CHANGED
@@ -2,6 +2,7 @@ import streamlit as st
|
|
2 |
import json
|
3 |
import pandas as pd
|
4 |
import os
|
|
|
5 |
|
6 |
st.set_page_config(page_title="Dataset Builder", layout="wide")
|
7 |
st.title("π JSONL Dataset Editor")
|
@@ -17,20 +18,24 @@ def get_all_fields(data):
|
|
17 |
all_keys.update(record.keys())
|
18 |
return sorted(all_keys)
|
19 |
|
20 |
-
|
|
|
|
|
|
|
|
|
|
|
21 |
if "data" not in st.session_state:
|
22 |
st.session_state.data = []
|
23 |
if "all_fields" not in st.session_state:
|
24 |
st.session_state.all_fields = []
|
25 |
-
if "
|
26 |
-
st.session_state.
|
27 |
|
28 |
-
# --- Load from temp if
|
29 |
-
if
|
30 |
with open(TMP_FILE, "r", encoding="utf-8") as f:
|
31 |
st.session_state.data = [json.loads(line) for line in f]
|
32 |
st.session_state.all_fields = get_all_fields(st.session_state.data)
|
33 |
-
st.session_state.prev_data = st.session_state.data.copy()
|
34 |
|
35 |
# --- Upload JSONL ---
|
36 |
uploaded_file = st.file_uploader("Upload a JSONL file", type=["jsonl"])
|
@@ -38,27 +43,22 @@ if uploaded_file:
|
|
38 |
content = uploaded_file.read().decode("utf-8")
|
39 |
st.session_state.data = [json.loads(line) for line in content.strip().splitlines()]
|
40 |
st.session_state.all_fields = get_all_fields(st.session_state.data)
|
41 |
-
|
42 |
-
|
43 |
-
for item in st.session_state.data:
|
44 |
-
f.write(json.dumps(item, ensure_ascii=False) + "\n")
|
45 |
st.rerun()
|
46 |
|
47 |
-
# ---
|
48 |
if not st.session_state.all_fields:
|
49 |
st.session_state.all_fields = ["context", "question", "answer"]
|
50 |
|
51 |
-
# --- Edit
|
52 |
st.markdown("### βοΈ Edit Records")
|
53 |
df = pd.DataFrame(st.session_state.data)
|
54 |
df = df.reindex(columns=st.session_state.all_fields)
|
55 |
|
56 |
-
# Ensure fields are strings for editor
|
57 |
for field in st.session_state.all_fields:
|
58 |
-
|
59 |
-
df[field] = df[field].astype(str)
|
60 |
|
61 |
-
# TextAreas for longer fields
|
62 |
column_configs = {
|
63 |
field: (
|
64 |
st.column_config.TextColumn(label=field, width="large")
|
@@ -70,35 +70,31 @@ column_configs = {
|
|
70 |
|
71 |
edited_df = st.data_editor(
|
72 |
df,
|
|
|
73 |
use_container_width=True,
|
74 |
num_rows="dynamic",
|
75 |
column_config=column_configs,
|
76 |
-
key="editable_table",
|
77 |
)
|
78 |
|
79 |
-
#
|
80 |
-
|
81 |
-
|
82 |
-
st.session_state.data
|
83 |
-
|
84 |
-
|
85 |
-
|
86 |
-
f.write(json.dumps(item, ensure_ascii=False) + "\n")
|
87 |
-
st.toast("β
Auto-saved!", icon="πΎ")
|
88 |
|
89 |
-
# --- Add
|
90 |
st.markdown("### β Add New Entry")
|
91 |
-
with st.form("
|
92 |
-
|
93 |
for field in st.session_state.all_fields:
|
94 |
-
|
95 |
submitted = st.form_submit_button("Add Entry")
|
96 |
if submitted:
|
97 |
-
st.session_state.data.append(
|
98 |
-
|
99 |
-
|
100 |
-
for item in st.session_state.data:
|
101 |
-
f.write(json.dumps(item, ensure_ascii=False) + "\n")
|
102 |
st.success("β
New entry added!")
|
103 |
st.rerun()
|
104 |
|
@@ -116,7 +112,6 @@ export_path = st.text_input("Save path", value="./exports/exported_dataset.jsonl
|
|
116 |
|
117 |
col1, col2, col3 = st.columns(3)
|
118 |
|
119 |
-
# Export
|
120 |
with col1:
|
121 |
if st.button("π Export JSONL"):
|
122 |
os.makedirs(os.path.dirname(export_path), exist_ok=True)
|
@@ -131,7 +126,6 @@ with col1:
|
|
131 |
st.session_state.clear()
|
132 |
st.rerun()
|
133 |
|
134 |
-
# Download temp
|
135 |
with col2:
|
136 |
if os.path.exists(TMP_FILE):
|
137 |
with open(TMP_FILE, "r", encoding="utf-8") as f:
|
@@ -142,10 +136,7 @@ with col2:
|
|
142 |
file_name="session_dataset.jsonl",
|
143 |
mime="application/json",
|
144 |
)
|
145 |
-
else:
|
146 |
-
st.warning("β οΈ No temp file found.")
|
147 |
|
148 |
-
# Clear session
|
149 |
with col3:
|
150 |
if st.button("ποΈ Clear Session"):
|
151 |
if os.path.exists(TMP_FILE):
|
|
|
2 |
import json
|
3 |
import pandas as pd
|
4 |
import os
|
5 |
+
from uuid import uuid4
|
6 |
|
7 |
st.set_page_config(page_title="Dataset Builder", layout="wide")
|
8 |
st.title("π JSONL Dataset Editor")
|
|
|
18 |
all_keys.update(record.keys())
|
19 |
return sorted(all_keys)
|
20 |
|
21 |
+
def save_to_file():
|
22 |
+
with open(TMP_FILE, "w", encoding="utf-8") as f:
|
23 |
+
for item in st.session_state.data:
|
24 |
+
f.write(json.dumps(item, ensure_ascii=False) + "\n")
|
25 |
+
|
26 |
+
# --- Initialize state ---
|
27 |
if "data" not in st.session_state:
|
28 |
st.session_state.data = []
|
29 |
if "all_fields" not in st.session_state:
|
30 |
st.session_state.all_fields = []
|
31 |
+
if "edit_key" not in st.session_state:
|
32 |
+
st.session_state.edit_key = str(uuid4())
|
33 |
|
34 |
+
# --- Load from temp if file exists ---
|
35 |
+
if os.path.exists(TMP_FILE) and not st.session_state.data:
|
36 |
with open(TMP_FILE, "r", encoding="utf-8") as f:
|
37 |
st.session_state.data = [json.loads(line) for line in f]
|
38 |
st.session_state.all_fields = get_all_fields(st.session_state.data)
|
|
|
39 |
|
40 |
# --- Upload JSONL ---
|
41 |
uploaded_file = st.file_uploader("Upload a JSONL file", type=["jsonl"])
|
|
|
43 |
content = uploaded_file.read().decode("utf-8")
|
44 |
st.session_state.data = [json.loads(line) for line in content.strip().splitlines()]
|
45 |
st.session_state.all_fields = get_all_fields(st.session_state.data)
|
46 |
+
save_to_file()
|
47 |
+
st.session_state.edit_key = str(uuid4())
|
|
|
|
|
48 |
st.rerun()
|
49 |
|
50 |
+
# --- Ensure default fields if none ---
|
51 |
if not st.session_state.all_fields:
|
52 |
st.session_state.all_fields = ["context", "question", "answer"]
|
53 |
|
54 |
+
# --- Edit Section ---
|
55 |
st.markdown("### βοΈ Edit Records")
|
56 |
df = pd.DataFrame(st.session_state.data)
|
57 |
df = df.reindex(columns=st.session_state.all_fields)
|
58 |
|
|
|
59 |
for field in st.session_state.all_fields:
|
60 |
+
df[field] = df[field].astype(str)
|
|
|
61 |
|
|
|
62 |
column_configs = {
|
63 |
field: (
|
64 |
st.column_config.TextColumn(label=field, width="large")
|
|
|
70 |
|
71 |
edited_df = st.data_editor(
|
72 |
df,
|
73 |
+
key=st.session_state.edit_key, # Unique key forces Streamlit to rerender editor
|
74 |
use_container_width=True,
|
75 |
num_rows="dynamic",
|
76 |
column_config=column_configs,
|
|
|
77 |
)
|
78 |
|
79 |
+
# If data changed, update state and save
|
80 |
+
if edited_df is not None:
|
81 |
+
new_data = edited_df.fillna("").to_dict(orient="records")
|
82 |
+
if new_data != st.session_state.data:
|
83 |
+
st.session_state.data = new_data
|
84 |
+
save_to_file()
|
85 |
+
st.toast("β
Changes auto-saved!", icon="πΎ")
|
|
|
|
|
86 |
|
87 |
+
# --- Add Entry ---
|
88 |
st.markdown("### β Add New Entry")
|
89 |
+
with st.form("add_form"):
|
90 |
+
new_item = {}
|
91 |
for field in st.session_state.all_fields:
|
92 |
+
new_item[field] = st.text_area(field, key=f"add_{field}")
|
93 |
submitted = st.form_submit_button("Add Entry")
|
94 |
if submitted:
|
95 |
+
st.session_state.data.append(new_item)
|
96 |
+
save_to_file()
|
97 |
+
st.session_state.edit_key = str(uuid4()) # Force editor to refresh
|
|
|
|
|
98 |
st.success("β
New entry added!")
|
99 |
st.rerun()
|
100 |
|
|
|
112 |
|
113 |
col1, col2, col3 = st.columns(3)
|
114 |
|
|
|
115 |
with col1:
|
116 |
if st.button("π Export JSONL"):
|
117 |
os.makedirs(os.path.dirname(export_path), exist_ok=True)
|
|
|
126 |
st.session_state.clear()
|
127 |
st.rerun()
|
128 |
|
|
|
129 |
with col2:
|
130 |
if os.path.exists(TMP_FILE):
|
131 |
with open(TMP_FILE, "r", encoding="utf-8") as f:
|
|
|
136 |
file_name="session_dataset.jsonl",
|
137 |
mime="application/json",
|
138 |
)
|
|
|
|
|
139 |
|
|
|
140 |
with col3:
|
141 |
if st.button("ποΈ Clear Session"):
|
142 |
if os.path.exists(TMP_FILE):
|