Maharshi Gor commited on
Commit
9b07040
·
1 Parent(s): 73d8b50

model_pipeline code refactor.

Browse files

* Made export pipeline btn invisible for now.

src/components/model_pipeline/model_pipeline.py CHANGED
@@ -71,6 +71,7 @@ class PipelineInterface:
71
  get_aux_states,
72
  inputs=[self.pipeline_state],
73
  outputs=[self.workflow_state, self.variables_state, self.output_panel_state],
 
74
  )
75
 
76
  # IO Variables
@@ -164,6 +165,45 @@ class PipelineInterface:
164
  )
165
  return add_step_btn
166
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
167
  def _render_output_panel(self, pipeline_state: PipelineState):
168
  dropdowns = {}
169
  available_variables = pipeline_state.workflow.get_available_variables()
@@ -201,28 +241,35 @@ class PipelineInterface:
201
  )
202
  return dropdowns
203
 
204
- def validate_workflow(self, state_dict: td.PipelineStateDict):
205
- """Validate the workflow."""
206
- try:
207
- state = self.sm.make_pipeline_state(state_dict)
208
- WorkflowValidator().validate(state.workflow)
209
- except ValueError as e:
210
- logger.exception(e)
211
- state_dict_str = yaml.dump(state_dict, default_flow_style=False, indent=2)
212
- logger.error(f"Could not validate workflow: \n{state_dict_str}")
213
- raise gr.Error(e)
214
 
215
- def _render_pipeline_header(self):
216
- # Add Step button at top
217
- input_variables_str = ", ".join([f"`{variable}`" for variable in self.input_variables])
218
- output_variables_str = ", ".join([f"`{variable}`" for variable in self.required_output_variables])
219
- if self.simple:
220
- instruction = "Create a simple single LLM call pipeline that takes in the following input variables and outputs the following output variables:"
221
- else:
222
- instruction = "Create a pipeline with the following input and output variables."
223
- gr.Markdown(f"### {instruction}")
224
- gr.Markdown(f"* Input Variables: {input_variables_str}")
225
- gr.Markdown(f"* Output Variables: {output_variables_str}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
226
 
227
  def render(self):
228
  """Render the pipeline UI."""
@@ -233,79 +280,44 @@ class PipelineInterface:
233
 
234
  # Function to render all steps
235
  @gr.render(
236
- triggers=[self.app.load, self.pipeline_change.change],
237
  inputs=[self.pipeline_state],
238
  concurrency_limit=1,
239
  concurrency_id="render_steps",
 
240
  )
241
- def render_steps(pipeline_state: td.PipelineStateDict, evt: gr.EventData):
242
  """Render all steps in the pipeline"""
243
  logger.info(
244
- f"Rerender triggered! \nInput Pipeline's UI State:{pipeline_state.get('ui_state')}\n Event: {evt.target} {evt._data}"
245
  )
246
- pipeline_state = self.sm.make_pipeline_state(pipeline_state)
247
- ui_state = pipeline_state.ui_state
248
- workflow = pipeline_state.workflow
249
- components = []
250
-
251
- step_objects = [] # Reset step objects list
252
- for i, step_id in enumerate(ui_state.step_ids):
253
- step_data = workflow.steps[step_id]
254
- step_ui_state = ui_state.steps[step_id]
255
- available_variables = pipeline_state.get_available_variables(step_id)
256
- sub_components = self._render_step(step_data, step_ui_state, available_variables, i, ui_state.n_steps)
257
- step_objects.append(sub_components)
258
-
259
- components.append(step_objects)
260
 
261
  if not self.simple:
262
  self._render_add_step_button(-1)
263
 
264
  @gr.render(
265
- triggers=[self.output_panel_state.change, self.app.load],
266
  inputs=[self.pipeline_state],
267
  concurrency_limit=1,
268
  concurrency_id="render_output_fields",
 
269
  )
270
  def render_output_fields(pipeline_state_dict: td.PipelineStateDict):
271
  pipeline_state = self.sm.make_pipeline_state(pipeline_state_dict)
272
  logger.debug(f"Rerendering output panel: {get_output_panel_state(pipeline_state.workflow)}")
273
  self._render_output_panel(pipeline_state)
274
 
275
- export_btn = gr.Button("Export Pipeline", elem_classes="export-button")
276
- # components.append(export_btn)
277
-
278
- # Add a code box to display the workflow JSON
279
- # with gr.Column(elem_classes="workflow-json-container"):
280
- with gr.Accordion(
281
- "Pipeline Preview (click to expand and edit)", open=False, elem_classes="pipeline-preview"
282
- ) as self.config_accordion:
283
- self.config_output = gr.Code(
284
- label="Workflow Configuration",
285
- show_label=False,
286
- language="yaml",
287
- elem_classes="workflow-json",
288
- interactive=True,
289
- autocomplete=True,
290
- )
291
- # components.append(config_accordion)
292
-
293
- self.config_output.blur(
294
- fn=self.sm.update_workflow_from_code,
295
- inputs=[self.config_output, self.pipeline_change],
296
- outputs=[self.pipeline_state, self.pipeline_change],
297
- )
298
-
299
- # Connect the export button to show the workflow JSON
300
- self.add_triggers_for_pipeline_export([export_btn.click], self.pipeline_state, scroll=True)
301
- export_btn.click(self.validate_workflow, inputs=[self.pipeline_state], outputs=[]).success(
302
- fn=lambda: gr.update(visible=True, open=True), outputs=[self.config_accordion]
303
- )
304
 
305
  def add_triggers_for_pipeline_export(self, triggers: list, input_pipeline_state: gr.State, scroll: bool = False):
306
  js = None
307
  if scroll:
308
  js = "() => {document.querySelector('.pipeline-preview').scrollIntoView({behavior: 'smooth'})}"
 
 
 
309
  gr.on(
310
  triggers,
311
  self.validate_workflow,
 
71
  get_aux_states,
72
  inputs=[self.pipeline_state],
73
  outputs=[self.workflow_state, self.variables_state, self.output_panel_state],
74
+ trigger_mode="multiple",
75
  )
76
 
77
  # IO Variables
 
165
  )
166
  return add_step_btn
167
 
168
+ def validate_workflow(self, state_dict: td.PipelineStateDict):
169
+ """Validate the workflow."""
170
+ try:
171
+ state = self.sm.make_pipeline_state(state_dict)
172
+ WorkflowValidator().validate(state.workflow)
173
+ except ValueError as e:
174
+ logger.exception(e)
175
+ state_dict_str = yaml.dump(state_dict, default_flow_style=False, indent=2)
176
+ logger.error(f"Could not validate workflow: \n{state_dict_str}")
177
+ raise gr.Error(e)
178
+
179
+ def _render_pipeline_header(self):
180
+ # Add Step button at top
181
+ input_variables_str = ", ".join([f"`{variable}`" for variable in self.input_variables])
182
+ output_variables_str = ", ".join([f"`{variable}`" for variable in self.required_output_variables])
183
+ if self.simple:
184
+ instruction = "Create a simple single LLM call pipeline that takes in the following input variables and outputs the following output variables:"
185
+ else:
186
+ instruction = "Create a pipeline with the following input and output variables."
187
+ gr.Markdown(f"### {instruction}")
188
+ gr.Markdown(f"* Input Variables: {input_variables_str}")
189
+ gr.Markdown(f"* Output Variables: {output_variables_str}")
190
+
191
+ def _render_model_steps(self, pipeline_state: PipelineState):
192
+ ui_state = pipeline_state.ui_state
193
+ workflow = pipeline_state.workflow
194
+ components = []
195
+
196
+ step_objects = [] # Reset step objects list
197
+ for i, step_id in enumerate(ui_state.step_ids):
198
+ step_data = workflow.steps[step_id]
199
+ step_ui_state = ui_state.steps[step_id]
200
+ available_variables = pipeline_state.get_available_variables(step_id)
201
+ sub_components = self._render_step(step_data, step_ui_state, available_variables, i, ui_state.n_steps)
202
+ step_objects.append(sub_components)
203
+
204
+ components.append(step_objects)
205
+ return components
206
+
207
  def _render_output_panel(self, pipeline_state: PipelineState):
208
  dropdowns = {}
209
  available_variables = pipeline_state.workflow.get_available_variables()
 
241
  )
242
  return dropdowns
243
 
244
+ def _render_pipeline_preview(self):
245
+ export_btn = gr.Button("Export Pipeline", elem_classes="export-button", visible=False)
246
+ # components.append(export_btn)
 
 
 
 
 
 
 
247
 
248
+ # Add a code box to display the workflow JSON
249
+ # with gr.Column(elem_classes="workflow-json-container"):
250
+ with gr.Accordion(
251
+ "Pipeline Preview (click to expand and edit)", open=False, elem_classes="pipeline-preview"
252
+ ) as self.config_accordion:
253
+ self.config_output = gr.Code(
254
+ show_label=False,
255
+ language="yaml",
256
+ elem_classes="workflow-json",
257
+ interactive=True,
258
+ autocomplete=True,
259
+ )
260
+ # components.append(config_accordion)
261
+
262
+ self.config_output.blur(
263
+ fn=self.sm.update_workflow_from_code,
264
+ inputs=[self.config_output, self.pipeline_change],
265
+ outputs=[self.pipeline_state, self.pipeline_change],
266
+ )
267
+
268
+ # Connect the export button to show the workflow JSON
269
+ self.add_triggers_for_pipeline_export([export_btn.click], self.pipeline_state, scroll=True)
270
+ export_btn.click(self.validate_workflow, inputs=[self.pipeline_state], outputs=[]).success(
271
+ fn=lambda: gr.update(visible=True, open=True), outputs=[self.config_accordion]
272
+ )
273
 
274
  def render(self):
275
  """Render the pipeline UI."""
 
280
 
281
  # Function to render all steps
282
  @gr.render(
283
+ triggers=[self.pipeline_change.change],
284
  inputs=[self.pipeline_state],
285
  concurrency_limit=1,
286
  concurrency_id="render_steps",
287
+ trigger_mode="multiple",
288
  )
289
+ def render_steps(pipeline_state_dict: td.PipelineStateDict, evt: gr.EventData):
290
  """Render all steps in the pipeline"""
291
  logger.info(
292
+ f"Rerender pipeline steps triggered! \nInput Pipeline's UI State:{pipeline_state_dict.get('ui_state')}\n Event: {evt.target} {evt._data}"
293
  )
294
+ pipeline_state = self.sm.make_pipeline_state(pipeline_state_dict)
295
+ self._render_model_steps(pipeline_state)
 
 
 
 
 
 
 
 
 
 
 
 
296
 
297
  if not self.simple:
298
  self._render_add_step_button(-1)
299
 
300
  @gr.render(
301
+ triggers=[self.output_panel_state.change],
302
  inputs=[self.pipeline_state],
303
  concurrency_limit=1,
304
  concurrency_id="render_output_fields",
305
+ trigger_mode="multiple",
306
  )
307
  def render_output_fields(pipeline_state_dict: td.PipelineStateDict):
308
  pipeline_state = self.sm.make_pipeline_state(pipeline_state_dict)
309
  logger.debug(f"Rerendering output panel: {get_output_panel_state(pipeline_state.workflow)}")
310
  self._render_output_panel(pipeline_state)
311
 
312
+ self._render_pipeline_preview()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
313
 
314
  def add_triggers_for_pipeline_export(self, triggers: list, input_pipeline_state: gr.State, scroll: bool = False):
315
  js = None
316
  if scroll:
317
  js = "() => {document.querySelector('.pipeline-preview').scrollIntoView({behavior: 'smooth'})}"
318
+
319
+ # TODO: modify this validate workflow to user input level and not executable label.
320
+ # (workflows that can be converted to UI interface, no logical validation)
321
  gr.on(
322
  triggers,
323
  self.validate_workflow,