Maharshi Gor
commited on
Commit
·
9b07040
1
Parent(s):
73d8b50
model_pipeline code refactor.
Browse files* Made export pipeline btn invisible for now.
src/components/model_pipeline/model_pipeline.py
CHANGED
@@ -71,6 +71,7 @@ class PipelineInterface:
|
|
71 |
get_aux_states,
|
72 |
inputs=[self.pipeline_state],
|
73 |
outputs=[self.workflow_state, self.variables_state, self.output_panel_state],
|
|
|
74 |
)
|
75 |
|
76 |
# IO Variables
|
@@ -164,6 +165,45 @@ class PipelineInterface:
|
|
164 |
)
|
165 |
return add_step_btn
|
166 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
167 |
def _render_output_panel(self, pipeline_state: PipelineState):
|
168 |
dropdowns = {}
|
169 |
available_variables = pipeline_state.workflow.get_available_variables()
|
@@ -201,28 +241,35 @@ class PipelineInterface:
|
|
201 |
)
|
202 |
return dropdowns
|
203 |
|
204 |
-
def
|
205 |
-
|
206 |
-
|
207 |
-
state = self.sm.make_pipeline_state(state_dict)
|
208 |
-
WorkflowValidator().validate(state.workflow)
|
209 |
-
except ValueError as e:
|
210 |
-
logger.exception(e)
|
211 |
-
state_dict_str = yaml.dump(state_dict, default_flow_style=False, indent=2)
|
212 |
-
logger.error(f"Could not validate workflow: \n{state_dict_str}")
|
213 |
-
raise gr.Error(e)
|
214 |
|
215 |
-
|
216 |
-
#
|
217 |
-
|
218 |
-
|
219 |
-
|
220 |
-
|
221 |
-
|
222 |
-
|
223 |
-
|
224 |
-
|
225 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
226 |
|
227 |
def render(self):
|
228 |
"""Render the pipeline UI."""
|
@@ -233,79 +280,44 @@ class PipelineInterface:
|
|
233 |
|
234 |
# Function to render all steps
|
235 |
@gr.render(
|
236 |
-
triggers=[self.
|
237 |
inputs=[self.pipeline_state],
|
238 |
concurrency_limit=1,
|
239 |
concurrency_id="render_steps",
|
|
|
240 |
)
|
241 |
-
def render_steps(
|
242 |
"""Render all steps in the pipeline"""
|
243 |
logger.info(
|
244 |
-
f"Rerender triggered! \nInput Pipeline's UI State:{
|
245 |
)
|
246 |
-
pipeline_state = self.sm.make_pipeline_state(
|
247 |
-
|
248 |
-
workflow = pipeline_state.workflow
|
249 |
-
components = []
|
250 |
-
|
251 |
-
step_objects = [] # Reset step objects list
|
252 |
-
for i, step_id in enumerate(ui_state.step_ids):
|
253 |
-
step_data = workflow.steps[step_id]
|
254 |
-
step_ui_state = ui_state.steps[step_id]
|
255 |
-
available_variables = pipeline_state.get_available_variables(step_id)
|
256 |
-
sub_components = self._render_step(step_data, step_ui_state, available_variables, i, ui_state.n_steps)
|
257 |
-
step_objects.append(sub_components)
|
258 |
-
|
259 |
-
components.append(step_objects)
|
260 |
|
261 |
if not self.simple:
|
262 |
self._render_add_step_button(-1)
|
263 |
|
264 |
@gr.render(
|
265 |
-
triggers=[self.output_panel_state.change
|
266 |
inputs=[self.pipeline_state],
|
267 |
concurrency_limit=1,
|
268 |
concurrency_id="render_output_fields",
|
|
|
269 |
)
|
270 |
def render_output_fields(pipeline_state_dict: td.PipelineStateDict):
|
271 |
pipeline_state = self.sm.make_pipeline_state(pipeline_state_dict)
|
272 |
logger.debug(f"Rerendering output panel: {get_output_panel_state(pipeline_state.workflow)}")
|
273 |
self._render_output_panel(pipeline_state)
|
274 |
|
275 |
-
|
276 |
-
# components.append(export_btn)
|
277 |
-
|
278 |
-
# Add a code box to display the workflow JSON
|
279 |
-
# with gr.Column(elem_classes="workflow-json-container"):
|
280 |
-
with gr.Accordion(
|
281 |
-
"Pipeline Preview (click to expand and edit)", open=False, elem_classes="pipeline-preview"
|
282 |
-
) as self.config_accordion:
|
283 |
-
self.config_output = gr.Code(
|
284 |
-
label="Workflow Configuration",
|
285 |
-
show_label=False,
|
286 |
-
language="yaml",
|
287 |
-
elem_classes="workflow-json",
|
288 |
-
interactive=True,
|
289 |
-
autocomplete=True,
|
290 |
-
)
|
291 |
-
# components.append(config_accordion)
|
292 |
-
|
293 |
-
self.config_output.blur(
|
294 |
-
fn=self.sm.update_workflow_from_code,
|
295 |
-
inputs=[self.config_output, self.pipeline_change],
|
296 |
-
outputs=[self.pipeline_state, self.pipeline_change],
|
297 |
-
)
|
298 |
-
|
299 |
-
# Connect the export button to show the workflow JSON
|
300 |
-
self.add_triggers_for_pipeline_export([export_btn.click], self.pipeline_state, scroll=True)
|
301 |
-
export_btn.click(self.validate_workflow, inputs=[self.pipeline_state], outputs=[]).success(
|
302 |
-
fn=lambda: gr.update(visible=True, open=True), outputs=[self.config_accordion]
|
303 |
-
)
|
304 |
|
305 |
def add_triggers_for_pipeline_export(self, triggers: list, input_pipeline_state: gr.State, scroll: bool = False):
|
306 |
js = None
|
307 |
if scroll:
|
308 |
js = "() => {document.querySelector('.pipeline-preview').scrollIntoView({behavior: 'smooth'})}"
|
|
|
|
|
|
|
309 |
gr.on(
|
310 |
triggers,
|
311 |
self.validate_workflow,
|
|
|
71 |
get_aux_states,
|
72 |
inputs=[self.pipeline_state],
|
73 |
outputs=[self.workflow_state, self.variables_state, self.output_panel_state],
|
74 |
+
trigger_mode="multiple",
|
75 |
)
|
76 |
|
77 |
# IO Variables
|
|
|
165 |
)
|
166 |
return add_step_btn
|
167 |
|
168 |
+
def validate_workflow(self, state_dict: td.PipelineStateDict):
|
169 |
+
"""Validate the workflow."""
|
170 |
+
try:
|
171 |
+
state = self.sm.make_pipeline_state(state_dict)
|
172 |
+
WorkflowValidator().validate(state.workflow)
|
173 |
+
except ValueError as e:
|
174 |
+
logger.exception(e)
|
175 |
+
state_dict_str = yaml.dump(state_dict, default_flow_style=False, indent=2)
|
176 |
+
logger.error(f"Could not validate workflow: \n{state_dict_str}")
|
177 |
+
raise gr.Error(e)
|
178 |
+
|
179 |
+
def _render_pipeline_header(self):
|
180 |
+
# Add Step button at top
|
181 |
+
input_variables_str = ", ".join([f"`{variable}`" for variable in self.input_variables])
|
182 |
+
output_variables_str = ", ".join([f"`{variable}`" for variable in self.required_output_variables])
|
183 |
+
if self.simple:
|
184 |
+
instruction = "Create a simple single LLM call pipeline that takes in the following input variables and outputs the following output variables:"
|
185 |
+
else:
|
186 |
+
instruction = "Create a pipeline with the following input and output variables."
|
187 |
+
gr.Markdown(f"### {instruction}")
|
188 |
+
gr.Markdown(f"* Input Variables: {input_variables_str}")
|
189 |
+
gr.Markdown(f"* Output Variables: {output_variables_str}")
|
190 |
+
|
191 |
+
def _render_model_steps(self, pipeline_state: PipelineState):
|
192 |
+
ui_state = pipeline_state.ui_state
|
193 |
+
workflow = pipeline_state.workflow
|
194 |
+
components = []
|
195 |
+
|
196 |
+
step_objects = [] # Reset step objects list
|
197 |
+
for i, step_id in enumerate(ui_state.step_ids):
|
198 |
+
step_data = workflow.steps[step_id]
|
199 |
+
step_ui_state = ui_state.steps[step_id]
|
200 |
+
available_variables = pipeline_state.get_available_variables(step_id)
|
201 |
+
sub_components = self._render_step(step_data, step_ui_state, available_variables, i, ui_state.n_steps)
|
202 |
+
step_objects.append(sub_components)
|
203 |
+
|
204 |
+
components.append(step_objects)
|
205 |
+
return components
|
206 |
+
|
207 |
def _render_output_panel(self, pipeline_state: PipelineState):
|
208 |
dropdowns = {}
|
209 |
available_variables = pipeline_state.workflow.get_available_variables()
|
|
|
241 |
)
|
242 |
return dropdowns
|
243 |
|
244 |
+
def _render_pipeline_preview(self):
|
245 |
+
export_btn = gr.Button("Export Pipeline", elem_classes="export-button", visible=False)
|
246 |
+
# components.append(export_btn)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
247 |
|
248 |
+
# Add a code box to display the workflow JSON
|
249 |
+
# with gr.Column(elem_classes="workflow-json-container"):
|
250 |
+
with gr.Accordion(
|
251 |
+
"Pipeline Preview (click to expand and edit)", open=False, elem_classes="pipeline-preview"
|
252 |
+
) as self.config_accordion:
|
253 |
+
self.config_output = gr.Code(
|
254 |
+
show_label=False,
|
255 |
+
language="yaml",
|
256 |
+
elem_classes="workflow-json",
|
257 |
+
interactive=True,
|
258 |
+
autocomplete=True,
|
259 |
+
)
|
260 |
+
# components.append(config_accordion)
|
261 |
+
|
262 |
+
self.config_output.blur(
|
263 |
+
fn=self.sm.update_workflow_from_code,
|
264 |
+
inputs=[self.config_output, self.pipeline_change],
|
265 |
+
outputs=[self.pipeline_state, self.pipeline_change],
|
266 |
+
)
|
267 |
+
|
268 |
+
# Connect the export button to show the workflow JSON
|
269 |
+
self.add_triggers_for_pipeline_export([export_btn.click], self.pipeline_state, scroll=True)
|
270 |
+
export_btn.click(self.validate_workflow, inputs=[self.pipeline_state], outputs=[]).success(
|
271 |
+
fn=lambda: gr.update(visible=True, open=True), outputs=[self.config_accordion]
|
272 |
+
)
|
273 |
|
274 |
def render(self):
|
275 |
"""Render the pipeline UI."""
|
|
|
280 |
|
281 |
# Function to render all steps
|
282 |
@gr.render(
|
283 |
+
triggers=[self.pipeline_change.change],
|
284 |
inputs=[self.pipeline_state],
|
285 |
concurrency_limit=1,
|
286 |
concurrency_id="render_steps",
|
287 |
+
trigger_mode="multiple",
|
288 |
)
|
289 |
+
def render_steps(pipeline_state_dict: td.PipelineStateDict, evt: gr.EventData):
|
290 |
"""Render all steps in the pipeline"""
|
291 |
logger.info(
|
292 |
+
f"Rerender pipeline steps triggered! \nInput Pipeline's UI State:{pipeline_state_dict.get('ui_state')}\n Event: {evt.target} {evt._data}"
|
293 |
)
|
294 |
+
pipeline_state = self.sm.make_pipeline_state(pipeline_state_dict)
|
295 |
+
self._render_model_steps(pipeline_state)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
296 |
|
297 |
if not self.simple:
|
298 |
self._render_add_step_button(-1)
|
299 |
|
300 |
@gr.render(
|
301 |
+
triggers=[self.output_panel_state.change],
|
302 |
inputs=[self.pipeline_state],
|
303 |
concurrency_limit=1,
|
304 |
concurrency_id="render_output_fields",
|
305 |
+
trigger_mode="multiple",
|
306 |
)
|
307 |
def render_output_fields(pipeline_state_dict: td.PipelineStateDict):
|
308 |
pipeline_state = self.sm.make_pipeline_state(pipeline_state_dict)
|
309 |
logger.debug(f"Rerendering output panel: {get_output_panel_state(pipeline_state.workflow)}")
|
310 |
self._render_output_panel(pipeline_state)
|
311 |
|
312 |
+
self._render_pipeline_preview()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
313 |
|
314 |
def add_triggers_for_pipeline_export(self, triggers: list, input_pipeline_state: gr.State, scroll: bool = False):
|
315 |
js = None
|
316 |
if scroll:
|
317 |
js = "() => {document.querySelector('.pipeline-preview').scrollIntoView({behavior: 'smooth'})}"
|
318 |
+
|
319 |
+
# TODO: modify this validate workflow to user input level and not executable label.
|
320 |
+
# (workflows that can be converted to UI interface, no logical validation)
|
321 |
gr.on(
|
322 |
triggers,
|
323 |
self.validate_workflow,
|