Update main_app.py
Browse files- main_app.py +547 -238
main_app.py
CHANGED
@@ -1,6 +1,8 @@
|
|
|
|
|
|
1 |
import marimo
|
2 |
|
3 |
-
__generated_with = "0.
|
4 |
app = marimo.App(width="medium")
|
5 |
|
6 |
|
@@ -11,17 +13,15 @@ def _():
|
|
11 |
return mo, os
|
12 |
|
13 |
|
14 |
-
@app.
|
15 |
-
def
|
16 |
-
|
17 |
-
|
18 |
-
|
19 |
-
return content
|
20 |
-
return (get_markdown_content,)
|
21 |
|
22 |
|
23 |
@app.cell
|
24 |
-
def _(
|
25 |
intro_text = get_markdown_content('intro_markdown/intro.md')
|
26 |
intro_marimo = get_markdown_content('intro_markdown/intro_marimo.md')
|
27 |
intro_notebook = get_markdown_content('intro_markdown/intro_notebook.md')
|
@@ -35,75 +35,130 @@ def _(get_markdown_content, mo):
|
|
35 |
])
|
36 |
|
37 |
mo.accordion({"## Notebook Introduction":intro})
|
38 |
-
return
|
39 |
|
40 |
|
41 |
@app.cell
|
42 |
def _(os):
|
43 |
### Imports
|
44 |
-
from typing import
|
45 |
-
|
46 |
-
)
|
47 |
from pathlib import Path
|
48 |
-
from urllib.request import urlopen
|
49 |
-
# from rich.markdown import Markdown as Markd
|
50 |
-
from rich.text import Text
|
51 |
-
from rich import print
|
52 |
-
from tqdm import tqdm
|
53 |
-
from enum import Enum
|
54 |
import pandas as pd
|
55 |
-
import
|
56 |
import requests
|
57 |
-
import
|
|
|
58 |
import urllib3
|
|
|
59 |
import base64
|
60 |
-
import time
|
61 |
-
import json
|
62 |
import uuid
|
63 |
import ssl
|
|
|
|
|
64 |
import ast
|
|
|
65 |
import re
|
66 |
|
67 |
-
pd.set_option('display.max_columns', None)
|
68 |
-
pd.set_option('display.max_rows', None)
|
69 |
-
pd.set_option('display.max_colwidth', None)
|
70 |
-
pd.set_option('display.width', None)
|
71 |
-
|
72 |
# Set explicit temporary directory
|
73 |
os.environ['TMPDIR'] = '/tmp/notebook_functions'
|
74 |
|
75 |
# Make sure Python's tempfile module also uses this directory
|
76 |
tempfile.tempdir = '/tmp/notebook_functions'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
77 |
return (
|
78 |
-
|
79 |
-
|
80 |
-
Enum,
|
81 |
-
List,
|
82 |
-
Optional,
|
83 |
-
Path,
|
84 |
-
Pattern,
|
85 |
-
Set,
|
86 |
-
Text,
|
87 |
-
Tuple,
|
88 |
-
Union,
|
89 |
ast,
|
90 |
-
|
91 |
-
getpass,
|
92 |
json,
|
93 |
pd,
|
94 |
-
|
95 |
-
re,
|
96 |
-
requests,
|
97 |
-
ssl,
|
98 |
tempfile,
|
99 |
-
time,
|
100 |
-
tqdm,
|
101 |
-
urllib3,
|
102 |
-
urlopen,
|
103 |
uuid,
|
104 |
)
|
105 |
|
106 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
107 |
@app.cell
|
108 |
def _(mo):
|
109 |
### Credentials for the watsonx.ai SDK client
|
@@ -133,90 +188,97 @@ def _(mo):
|
|
133 |
.batch(
|
134 |
wx_region = mo.ui.dropdown(regions, label="Select your watsonx.ai region:", value="US", searchable=True),
|
135 |
wx_api_key = mo.ui.text(placeholder="Add your IBM Cloud api-key...", label="IBM Cloud Api-key:", kind="password"),
|
136 |
-
|
137 |
space_id = mo.ui.text(placeholder="Add your watsonx.ai space_id...", label="Space_ID:", kind="text")
|
138 |
,)
|
139 |
.form(show_clear_button=True, bordered=False)
|
140 |
)
|
141 |
|
142 |
-
|
143 |
-
# client_instantiation_form
|
144 |
-
return client_instantiation_form, regions, wx_platform_url
|
145 |
|
146 |
|
147 |
@app.cell
|
148 |
-
def _(
|
149 |
-
|
150 |
-
|
151 |
-
|
152 |
-
|
153 |
-
|
154 |
-
|
155 |
-
|
156 |
-
|
157 |
-
|
158 |
-
cred_id = deployment_client.task_credentials.get_id(cred)
|
159 |
-
deployment_client.task_credentials.delete(cred_id)
|
160 |
-
|
161 |
-
# Store new credentials
|
162 |
-
return deployment_client.task_credentials.store()
|
163 |
-
|
164 |
-
if client_instantiation_form.value:
|
165 |
### Instantiate the watsonx.ai client
|
|
|
166 |
wx_credentials = Credentials(
|
167 |
-
url=
|
168 |
-
api_key=
|
169 |
)
|
170 |
|
171 |
-
|
172 |
-
|
|
|
|
|
173 |
|
174 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
175 |
else:
|
176 |
-
|
|
|
177 |
deployment_client = None
|
178 |
task_credentials_details = None
|
179 |
|
180 |
-
|
181 |
-
|
182 |
-
if deployment_client is not None:
|
183 |
client_callout_kind = "success"
|
184 |
else:
|
185 |
client_callout_kind = "neutral"
|
|
|
186 |
|
187 |
-
client_callout = mo.callout(template_variant, kind=client_callout_kind)
|
188 |
|
189 |
-
|
190 |
-
|
191 |
-
|
192 |
-
|
193 |
-
|
194 |
-
|
195 |
-
|
196 |
-
|
197 |
-
|
198 |
-
|
199 |
-
|
200 |
-
|
|
|
|
|
|
|
201 |
|
202 |
|
203 |
@app.cell
|
204 |
def _(
|
205 |
-
|
206 |
-
client_instantiation_form,
|
207 |
deploy_fnc,
|
208 |
deployment_definition,
|
209 |
fm,
|
210 |
function_editor,
|
211 |
hw_selection_table,
|
212 |
mo,
|
|
|
|
|
213 |
purge_tabs,
|
214 |
sc_m,
|
215 |
schema_editors,
|
216 |
selection_table,
|
|
|
217 |
upload_func,
|
|
|
218 |
):
|
219 |
-
|
220 |
###**Instantiate your watsonx.ai client:**
|
221 |
|
222 |
1. Select a region from the dropdown menu
|
@@ -229,11 +291,7 @@ def _(
|
|
229 |
|
230 |
---
|
231 |
|
232 |
-
{
|
233 |
-
|
234 |
-
---
|
235 |
-
|
236 |
-
{client_callout}
|
237 |
|
238 |
''')
|
239 |
|
@@ -246,7 +304,7 @@ def _(
|
|
246 |
}
|
247 |
)
|
248 |
|
249 |
-
|
250 |
|
251 |
1. Use the code editor window to create a function to deploy
|
252 |
<br>
|
@@ -285,7 +343,7 @@ def _(
|
|
285 |
|
286 |
''')
|
287 |
|
288 |
-
|
289 |
###**Review and Upload your function**
|
290 |
|
291 |
1. Review the function metadata specs JSON
|
@@ -306,7 +364,7 @@ def _(
|
|
306 |
|
307 |
''')
|
308 |
|
309 |
-
|
310 |
###**Deploy your function:**
|
311 |
|
312 |
1. Select a hardware specification (vCPUs/GB) that you want your function deployed on
|
@@ -339,32 +397,113 @@ def _(
|
|
339 |
|
340 |
''')
|
341 |
|
342 |
-
|
343 |
###**Helper Purge Functions:**
|
344 |
-
|
345 |
These functions help you retrieve, select and delete deployments, data assets or repository assets (functions, models, etc.) that you have in the deployment space. This is meant to support fast cleanup.
|
346 |
-
|
347 |
Select the tab based on what you want to delete, then click each of the buttons one by one after the previous gives a response.
|
348 |
-
|
349 |
---
|
350 |
|
351 |
{purge_tabs}
|
352 |
|
353 |
''')
|
354 |
|
355 |
-
|
356 |
-
|
357 |
-
|
358 |
-
|
359 |
-
|
360 |
-
|
361 |
-
|
362 |
-
|
363 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
364 |
)
|
365 |
|
366 |
-
|
367 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
368 |
|
369 |
|
370 |
@app.cell
|
@@ -394,7 +533,7 @@ def _(mo, template_variant):
|
|
394 |
|
395 |
return score
|
396 |
|
397 |
-
|
398 |
'''
|
399 |
|
400 |
function_editor = (
|
@@ -405,17 +544,17 @@ def _(mo, template_variant):
|
|
405 |
|
406 |
''')
|
407 |
.batch(
|
408 |
-
editor = mo.ui.code_editor(value=template, language="python", min_height=
|
409 |
)
|
410 |
.form(show_clear_button=True, bordered=False)
|
411 |
)
|
412 |
|
413 |
# function_editor
|
414 |
-
return
|
415 |
|
416 |
|
417 |
@app.cell
|
418 |
-
def _(function_editor, mo, os
|
419 |
function_name = None
|
420 |
if function_editor.value:
|
421 |
# Get the edited code from the function editor
|
@@ -429,12 +568,12 @@ def _(function_editor, mo, os, ast):
|
|
429 |
if isinstance(node, ast.FunctionDef):
|
430 |
function_name = node.name
|
431 |
break
|
432 |
-
|
433 |
if function_name is not None:
|
434 |
# Set deployable_function to None since we're not executing the code
|
435 |
deployable_function = None
|
436 |
mo.md(f"Found function: '{function_name}' (using file path approach)")
|
437 |
-
|
438 |
# Create the directory if it doesn't exist
|
439 |
save_dir = "/tmp/notebook_functions"
|
440 |
os.makedirs(save_dir, exist_ok=True)
|
@@ -446,18 +585,238 @@ def _(function_editor, mo, os, ast):
|
|
446 |
mo.md("No function found in the editor code")
|
447 |
except SyntaxError as e:
|
448 |
mo.md(f"Syntax error in function code: {str(e)}")
|
449 |
-
return (
|
450 |
-
|
451 |
-
|
452 |
-
|
453 |
-
|
454 |
-
|
455 |
-
|
456 |
-
|
457 |
-
|
458 |
-
|
459 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
460 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
461 |
|
462 |
|
463 |
@app.cell
|
@@ -538,29 +897,23 @@ def _(deployment_client, mo, pd):
|
|
538 |
initial_selection=[0]
|
539 |
)
|
540 |
|
541 |
-
return (
|
542 |
-
framework_mapping,
|
543 |
-
preferred_order,
|
544 |
-
sel_df,
|
545 |
-
selection_table,
|
546 |
-
supported_specs,
|
547 |
-
)
|
548 |
|
549 |
|
550 |
@app.cell
|
551 |
def _(mo):
|
552 |
input_schema_checkbox = mo.ui.checkbox(label="Add input schema (optional)")
|
553 |
output_schema_checkbox = mo.ui.checkbox(label="Add output schema (optional)")
|
554 |
-
sample_input_checkbox = mo.ui.checkbox(label="Add sample input example (optional)")
|
555 |
-
return input_schema_checkbox, output_schema_checkbox
|
556 |
|
557 |
|
558 |
@app.cell
|
559 |
def _(
|
|
|
560 |
input_schema_checkbox,
|
561 |
mo,
|
562 |
output_schema_checkbox,
|
563 |
-
sample_input_checkbox,
|
564 |
selection_table,
|
565 |
template_variant,
|
566 |
):
|
@@ -604,7 +957,7 @@ def _(
|
|
604 |
schema_metadata=mo.hstack([
|
605 |
input_schema_checkbox,
|
606 |
output_schema_checkbox,
|
607 |
-
sample_input_checkbox
|
608 |
],
|
609 |
justify="center", gap=1, align="center", wrap=True
|
610 |
)
|
@@ -625,10 +978,7 @@ def _(
|
|
625 |
return (
|
626 |
description_input,
|
627 |
fm,
|
628 |
-
fnc_nm,
|
629 |
-
func_metadata,
|
630 |
sc_m,
|
631 |
-
schema_metadata,
|
632 |
software_spec,
|
633 |
tags_editor,
|
634 |
uploaded_function_name,
|
@@ -638,7 +988,7 @@ def _(
|
|
638 |
@app.cell
|
639 |
def _(json, mo, template_variant):
|
640 |
if template_variant.value == "Stream Files to IBM COS [Example]":
|
641 |
-
from cos_stream_schema_examples import input_schema, output_schema
|
642 |
else:
|
643 |
input_schema = [
|
644 |
{
|
@@ -676,40 +1026,20 @@ def _(json, mo, template_variant):
|
|
676 |
}
|
677 |
]
|
678 |
|
679 |
-
sample_input = {
|
680 |
-
'input_data': [
|
681 |
-
{
|
682 |
-
'fields': ['<variable name 1>', '<variable name 2>'],
|
683 |
-
'values': [
|
684 |
-
['<sample input value for variable 1>', '<sample input value for variable 2>']
|
685 |
-
]
|
686 |
-
}
|
687 |
-
]
|
688 |
-
}
|
689 |
|
690 |
|
691 |
-
input_schema_editor = mo.ui.code_editor(value=json.dumps(input_schema, indent=4), language="python", min_height=
|
692 |
-
output_schema_editor = mo.ui.code_editor(value=json.dumps(output_schema, indent=4), language="python", min_height=
|
693 |
-
sample_input_editor = mo.ui.code_editor(value=json.dumps(sample_input, indent=4), language="python", min_height=25)
|
694 |
|
695 |
schema_editors = mo.accordion(
|
696 |
{
|
697 |
"""**Input Schema Metadata Editor**""": input_schema_editor,
|
698 |
"""**Output Schema Metadata Editor**""": output_schema_editor,
|
699 |
-
"""**Sample Input Metadata Editor**""": sample_input_editor
|
700 |
}, multiple=True
|
701 |
)
|
702 |
|
703 |
# schema_editors
|
704 |
-
return
|
705 |
-
input_schema,
|
706 |
-
input_schema_editor,
|
707 |
-
output_schema,
|
708 |
-
output_schema_editor,
|
709 |
-
sample_input,
|
710 |
-
sample_input_editor,
|
711 |
-
schema_editors,
|
712 |
-
)
|
713 |
|
714 |
|
715 |
@app.cell
|
@@ -725,8 +1055,6 @@ def _(
|
|
725 |
os,
|
726 |
output_schema_checkbox,
|
727 |
output_schema_editor,
|
728 |
-
sample_input_checkbox,
|
729 |
-
sample_input_editor,
|
730 |
selection_table,
|
731 |
software_spec,
|
732 |
tags_editor,
|
@@ -771,12 +1099,12 @@ def _(
|
|
771 |
function_meta[deployment_client.repository.FunctionMetaNames.OUTPUT_DATA_SCHEMAS] = ast.literal_eval(output_schema_editor.value)
|
772 |
|
773 |
# Add sample input if checkbox is checked
|
774 |
-
if sample_input_checkbox.value:
|
775 |
-
|
776 |
-
|
777 |
-
|
778 |
-
|
779 |
-
|
780 |
|
781 |
def upload_function(function_meta, use_function_object=False):
|
782 |
"""
|
@@ -830,7 +1158,7 @@ def _(
|
|
830 |
# # Upload using the absolute file path
|
831 |
# abs_file_path = os.path.abspath(file_path)
|
832 |
# mo.md(f"Uploading function from file: {abs_file_path}")
|
833 |
-
|
834 |
# # Using the absolute file path might help in some environments
|
835 |
# func_details = deployment_client.repository.store_function(abs_file_path, function_meta)
|
836 |
# Upload using the file path approach
|
@@ -838,15 +1166,15 @@ def _(
|
|
838 |
# Create a zip file of the Python module
|
839 |
import gzip
|
840 |
import shutil
|
841 |
-
|
842 |
# Path for the gzipped file
|
843 |
gz_path = f"{save_dir}/{func_name}.py.gz"
|
844 |
-
|
845 |
# Create gzip file
|
846 |
with open(file_path, 'rb') as f_in:
|
847 |
with gzip.open(gz_path, 'wb') as f_out:
|
848 |
shutil.copyfileobj(f_in, f_out)
|
849 |
-
|
850 |
# Upload using the gzipped file path
|
851 |
mo.md(f"Uploading function from gzip: {gz_path}")
|
852 |
func_details = deployment_client.repository.store_function(gz_path, function_meta)
|
@@ -874,15 +1202,7 @@ def _(
|
|
874 |
)
|
875 |
|
876 |
# function_meta
|
877 |
-
return
|
878 |
-
filtered_tags,
|
879 |
-
function_meta,
|
880 |
-
get_upload_status,
|
881 |
-
set_upload_status,
|
882 |
-
upload_button,
|
883 |
-
upload_function,
|
884 |
-
upload_status,
|
885 |
-
)
|
886 |
|
887 |
|
888 |
@app.cell
|
@@ -899,7 +1219,7 @@ def _(get_upload_status, mo, upload_button):
|
|
899 |
upload_button,
|
900 |
mo.md(f"**Status:** {get_upload_status()}")
|
901 |
], justify="space-around", align="center")
|
902 |
-
return artifact_id, upload_func
|
903 |
|
904 |
|
905 |
@app.cell
|
@@ -982,16 +1302,7 @@ def _(deployment_client, mo, pd, upload_button, uuid):
|
|
982 |
initial_selection=[0]
|
983 |
)
|
984 |
|
985 |
-
return
|
986 |
-
deployment_name,
|
987 |
-
deployment_type,
|
988 |
-
hardware_specs,
|
989 |
-
hardware_specs_df,
|
990 |
-
hw_df,
|
991 |
-
hw_selection_table,
|
992 |
-
reorder_hardware_specifications,
|
993 |
-
uuid_suffix,
|
994 |
-
)
|
995 |
|
996 |
|
997 |
@app.cell
|
@@ -1003,7 +1314,6 @@ def _(
|
|
1003 |
deployment_type,
|
1004 |
hw_selection_table,
|
1005 |
mo,
|
1006 |
-
print,
|
1007 |
upload_button,
|
1008 |
):
|
1009 |
def deploy_function(artifact_id, deployment_type):
|
@@ -1082,15 +1392,7 @@ def _(
|
|
1082 |
], justify="space-around")
|
1083 |
|
1084 |
# deployment_definition
|
1085 |
-
return
|
1086 |
-
deploy_button,
|
1087 |
-
deploy_function,
|
1088 |
-
deployment_definition,
|
1089 |
-
deployment_status,
|
1090 |
-
get_deployment_id,
|
1091 |
-
get_deployment_info,
|
1092 |
-
selected_hw_config,
|
1093 |
-
)
|
1094 |
|
1095 |
|
1096 |
@app.cell
|
@@ -1106,7 +1408,7 @@ def _(deploy_button, deployment_definition, mo):
|
|
1106 |
|
1107 |
|
1108 |
@app.cell(hide_code=True)
|
1109 |
-
def _(deployment_client, mo):
|
1110 |
### Functions to List , Get ID's as a list and Purge of Assets
|
1111 |
|
1112 |
def get_deployment_list():
|
@@ -1141,7 +1443,7 @@ def _(deployment_client, mo):
|
|
1141 |
return repository_list
|
1142 |
|
1143 |
#----
|
1144 |
-
|
1145 |
def delete_with_progress(ids_list, delete_function, item_type="items"):
|
1146 |
"""
|
1147 |
Generic wrapper that adds a progress bar to any deletion function
|
@@ -1188,7 +1490,6 @@ def _(deployment_client, mo):
|
|
1188 |
delete_data_assets,
|
1189 |
delete_deployments,
|
1190 |
delete_repository_items,
|
1191 |
-
delete_with_progress,
|
1192 |
get_data_asset_ids,
|
1193 |
get_data_assets_list,
|
1194 |
get_deployment_ids,
|
@@ -1197,6 +1498,7 @@ def _(deployment_client, mo):
|
|
1197 |
get_repository_list,
|
1198 |
)
|
1199 |
|
|
|
1200 |
@app.cell
|
1201 |
def _(get_data_assets_tab, get_deployments_tab, get_repository_tab, mo):
|
1202 |
if get_deployments_tab() is not None:
|
@@ -1213,47 +1515,54 @@ def _(get_data_assets_tab, get_deployments_tab, get_repository_tab, mo):
|
|
1213 |
data_assets_table = mo.ui.table(get_data_assets_tab())
|
1214 |
else:
|
1215 |
data_assets_table = mo.md("No Table Loaded")
|
1216 |
-
|
1217 |
return data_assets_table, deployments_table, repository_table
|
1218 |
|
|
|
1219 |
@app.cell
|
1220 |
-
def _(
|
|
|
|
|
|
|
|
|
|
|
|
|
1221 |
deployments_purge_stack = mo.hstack([get_deployments_button, get_deployment_id_list, purge_deployments])
|
1222 |
deployments_purge_stack_results = mo.vstack([deployments_table, get_deployment_id_list.value, purge_deployments.value])
|
1223 |
|
1224 |
deployments_purge_tab = mo.vstack([deployments_purge_stack, deployments_purge_stack_results])
|
1225 |
-
return (
|
1226 |
-
deployments_purge_stack,
|
1227 |
-
deployments_purge_stack_results,
|
1228 |
-
deployments_purge_tab,
|
1229 |
-
)
|
1230 |
|
1231 |
|
1232 |
@app.cell
|
1233 |
-
def _(
|
|
|
|
|
|
|
|
|
|
|
|
|
1234 |
repository_purge_stack = mo.hstack([get_repository_button, get_repository_id_list, purge_repository])
|
1235 |
|
1236 |
repository_purge_stack_results = mo.vstack([repository_table, get_repository_id_list.value, purge_repository.value])
|
1237 |
|
1238 |
repository_purge_tab = mo.vstack([repository_purge_stack, repository_purge_stack_results])
|
1239 |
-
return (
|
1240 |
-
repository_purge_stack,
|
1241 |
-
repository_purge_stack_results,
|
1242 |
-
repository_purge_tab,
|
1243 |
-
)
|
1244 |
|
1245 |
|
1246 |
@app.cell
|
1247 |
-
def _(
|
|
|
|
|
|
|
|
|
|
|
|
|
1248 |
data_assets_purge_stack = mo.hstack([get_data_assets_button, get_data_asset_id_list, purge_data_assets])
|
1249 |
data_assets_purge_stack_results = mo.vstack([data_assets_table, get_data_asset_id_list.value, purge_data_assets.value])
|
1250 |
|
1251 |
data_assets_purge_tab = mo.vstack([data_assets_purge_stack, data_assets_purge_stack_results])
|
1252 |
-
return (
|
1253 |
-
data_assets_purge_stack,
|
1254 |
-
data_assets_purge_stack_results,
|
1255 |
-
data_assets_purge_tab,
|
1256 |
-
)
|
1257 |
|
1258 |
|
1259 |
@app.cell
|
@@ -1262,9 +1571,9 @@ def _(data_assets_purge_tab, deployments_purge_tab, mo, repository_purge_tab):
|
|
1262 |
{"Purge Deployments": deployments_purge_tab, "Purge Repository Assets": repository_purge_tab,"Purge Data Assets": data_assets_purge_tab }, lazy=False
|
1263 |
)
|
1264 |
|
1265 |
-
# asset_purge
|
1266 |
return (purge_tabs,)
|
1267 |
|
|
|
1268 |
@app.cell
|
1269 |
def _(mo):
|
1270 |
get_deployments_tab, set_deployments_tab = mo.state(None)
|
|
|
1 |
+
|
2 |
+
|
3 |
import marimo
|
4 |
|
5 |
+
__generated_with = "0.13.0"
|
6 |
app = marimo.App(width="medium")
|
7 |
|
8 |
|
|
|
13 |
return mo, os
|
14 |
|
15 |
|
16 |
+
@app.function
|
17 |
+
def get_markdown_content(file_path):
|
18 |
+
with open(file_path, 'r', encoding='utf-8') as file:
|
19 |
+
content = file.read()
|
20 |
+
return content
|
|
|
|
|
21 |
|
22 |
|
23 |
@app.cell
|
24 |
+
def _(mo):
|
25 |
intro_text = get_markdown_content('intro_markdown/intro.md')
|
26 |
intro_marimo = get_markdown_content('intro_markdown/intro_marimo.md')
|
27 |
intro_notebook = get_markdown_content('intro_markdown/intro_notebook.md')
|
|
|
35 |
])
|
36 |
|
37 |
mo.accordion({"## Notebook Introduction":intro})
|
38 |
+
return
|
39 |
|
40 |
|
41 |
@app.cell
|
42 |
def _(os):
|
43 |
### Imports
|
44 |
+
from typing import Any, Dict, List, Optional, Pattern, Set, Union, Tuple
|
45 |
+
from ibm_watsonx_ai import APIClient, Credentials
|
|
|
46 |
from pathlib import Path
|
|
|
|
|
|
|
|
|
|
|
|
|
47 |
import pandas as pd
|
48 |
+
import mimetypes
|
49 |
import requests
|
50 |
+
import zipfile
|
51 |
+
import polars
|
52 |
import urllib3
|
53 |
+
import tempfile
|
54 |
import base64
|
|
|
|
|
55 |
import uuid
|
56 |
import ssl
|
57 |
+
import time
|
58 |
+
import json
|
59 |
import ast
|
60 |
+
import io
|
61 |
import re
|
62 |
|
|
|
|
|
|
|
|
|
|
|
63 |
# Set explicit temporary directory
|
64 |
os.environ['TMPDIR'] = '/tmp/notebook_functions'
|
65 |
|
66 |
# Make sure Python's tempfile module also uses this directory
|
67 |
tempfile.tempdir = '/tmp/notebook_functions'
|
68 |
+
|
69 |
+
def get_iam_token(api_key):
|
70 |
+
return requests.post(
|
71 |
+
'https://iam.cloud.ibm.com/identity/token',
|
72 |
+
headers={'Content-Type': 'application/x-www-form-urlencoded'},
|
73 |
+
data={'grant_type': 'urn:ibm:params:oauth:grant-type:apikey', 'apikey': api_key}
|
74 |
+
).json()['access_token']
|
75 |
+
|
76 |
+
def setup_task_credentials(client):
|
77 |
+
# Get existing task credentials
|
78 |
+
existing_credentials = client.task_credentials.get_details()
|
79 |
+
|
80 |
+
# Delete existing credentials if any
|
81 |
+
if "resources" in existing_credentials and existing_credentials["resources"]:
|
82 |
+
for cred in existing_credentials["resources"]:
|
83 |
+
cred_id = client.task_credentials.get_id(cred)
|
84 |
+
client.task_credentials.delete(cred_id)
|
85 |
+
|
86 |
+
# Store new credentials
|
87 |
+
return client.task_credentials.store()
|
88 |
+
|
89 |
+
def get_cred_value(key, creds_var_name="baked_in_creds", default=""): ### Helper for working with preset credentials
|
90 |
+
"""
|
91 |
+
Helper function to safely get a value from a credentials dictionary.
|
92 |
+
|
93 |
+
Args:
|
94 |
+
key: The key to look up in the credentials dictionary.
|
95 |
+
creds_var_name: The variable name of the credentials dictionary.
|
96 |
+
default: The default value to return if the key is not found.
|
97 |
+
|
98 |
+
Returns:
|
99 |
+
The value from the credentials dictionary if it exists and contains the key,
|
100 |
+
otherwise returns the default value.
|
101 |
+
"""
|
102 |
+
# Check if the credentials variable exists in globals
|
103 |
+
if creds_var_name in globals():
|
104 |
+
creds_dict = globals()[creds_var_name]
|
105 |
+
if isinstance(creds_dict, dict) and key in creds_dict:
|
106 |
+
return creds_dict[key]
|
107 |
+
return default
|
108 |
return (
|
109 |
+
APIClient,
|
110 |
+
Credentials,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
111 |
ast,
|
112 |
+
get_iam_token,
|
|
|
113 |
json,
|
114 |
pd,
|
115 |
+
setup_task_credentials,
|
|
|
|
|
|
|
116 |
tempfile,
|
|
|
|
|
|
|
|
|
117 |
uuid,
|
118 |
)
|
119 |
|
120 |
|
121 |
+
@app.cell
|
122 |
+
def _(client_instantiation_form, os):
|
123 |
+
if client_instantiation_form.value:
|
124 |
+
client_setup = client_instantiation_form.value
|
125 |
+
else:
|
126 |
+
client_setup = None
|
127 |
+
|
128 |
+
### Extract Credential Variables:
|
129 |
+
if client_setup is not None:
|
130 |
+
wx_url = client_setup["wx_region"]
|
131 |
+
wx_api_key = client_setup["wx_api_key"]
|
132 |
+
os.environ["WATSONX_APIKEY"] = wx_api_key
|
133 |
+
|
134 |
+
if client_setup["project_id"] is not None:
|
135 |
+
project_id = client_setup["project_id"]
|
136 |
+
else:
|
137 |
+
project_id = None
|
138 |
+
|
139 |
+
if client_setup["space_id"] is not None:
|
140 |
+
space_id = client_setup["space_id"]
|
141 |
+
else:
|
142 |
+
space_id = None
|
143 |
+
|
144 |
+
else:
|
145 |
+
os.environ["WATSONX_APIKEY"] = ""
|
146 |
+
project_id = None
|
147 |
+
space_id = None
|
148 |
+
wx_api_key = None
|
149 |
+
wx_url = None
|
150 |
+
return client_setup, project_id, space_id, wx_api_key, wx_url
|
151 |
+
|
152 |
+
|
153 |
+
@app.cell
|
154 |
+
def _(client_setup, get_iam_token, wx_api_key):
|
155 |
+
if client_setup:
|
156 |
+
token = get_iam_token(wx_api_key)
|
157 |
+
else:
|
158 |
+
token = None
|
159 |
+
return
|
160 |
+
|
161 |
+
|
162 |
@app.cell
|
163 |
def _(mo):
|
164 |
### Credentials for the watsonx.ai SDK client
|
|
|
188 |
.batch(
|
189 |
wx_region = mo.ui.dropdown(regions, label="Select your watsonx.ai region:", value="US", searchable=True),
|
190 |
wx_api_key = mo.ui.text(placeholder="Add your IBM Cloud api-key...", label="IBM Cloud Api-key:", kind="password"),
|
191 |
+
project_id = mo.ui.text(placeholder="Add your watsonx.ai project_id...", label="Project_ID:", kind="text"),
|
192 |
space_id = mo.ui.text(placeholder="Add your watsonx.ai space_id...", label="Space_ID:", kind="text")
|
193 |
,)
|
194 |
.form(show_clear_button=True, bordered=False)
|
195 |
)
|
196 |
|
197 |
+
return (client_instantiation_form,)
|
|
|
|
|
198 |
|
199 |
|
200 |
@app.cell
|
201 |
+
def _(
|
202 |
+
APIClient,
|
203 |
+
Credentials,
|
204 |
+
client_setup,
|
205 |
+
project_id,
|
206 |
+
setup_task_credentials,
|
207 |
+
space_id,
|
208 |
+
wx_api_key,
|
209 |
+
wx_url,
|
210 |
+
):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
211 |
### Instantiate the watsonx.ai client
|
212 |
+
if client_setup:
|
213 |
wx_credentials = Credentials(
|
214 |
+
url=wx_url,
|
215 |
+
api_key=wx_api_key
|
216 |
)
|
217 |
|
218 |
+
if project_id:
|
219 |
+
project_client = APIClient(credentials=wx_credentials, project_id=project_id)
|
220 |
+
else:
|
221 |
+
project_client = None
|
222 |
|
223 |
+
if space_id:
|
224 |
+
deployment_client = APIClient(credentials=wx_credentials, space_id=space_id)
|
225 |
+
else:
|
226 |
+
deployment_client = None
|
227 |
+
|
228 |
+
if project_client is not None:
|
229 |
+
task_credentials_details = setup_task_credentials(project_client)
|
230 |
+
else:
|
231 |
+
task_credentials_details = setup_task_credentials(deployment_client)
|
232 |
else:
|
233 |
+
wx_credentials = None
|
234 |
+
project_client = None
|
235 |
deployment_client = None
|
236 |
task_credentials_details = None
|
237 |
|
238 |
+
if project_client is not None or deployment_client is not None:
|
|
|
|
|
239 |
client_callout_kind = "success"
|
240 |
else:
|
241 |
client_callout_kind = "neutral"
|
242 |
+
return client_callout_kind, deployment_client
|
243 |
|
|
|
244 |
|
245 |
+
@app.cell
|
246 |
+
def _(mo):
|
247 |
+
template_variants = [
|
248 |
+
"Base",
|
249 |
+
"Stream Files to IBM COS [Example]",
|
250 |
+
]
|
251 |
+
template_variant = mo.ui.dropdown(template_variants, label="Code Template:", value="Base")
|
252 |
+
return (template_variant,)
|
253 |
+
|
254 |
+
|
255 |
+
@app.cell
|
256 |
+
def _(client_callout_kind, client_instantiation_form, mo, template_variant):
|
257 |
+
client_callout = mo.callout(template_variant, kind=client_callout_kind)
|
258 |
+
client_stack = mo.hstack([client_instantiation_form, client_callout], align="center", justify="space-around")
|
259 |
+
return (client_stack,)
|
260 |
|
261 |
|
262 |
@app.cell
|
263 |
def _(
|
264 |
+
client_stack,
|
|
|
265 |
deploy_fnc,
|
266 |
deployment_definition,
|
267 |
fm,
|
268 |
function_editor,
|
269 |
hw_selection_table,
|
270 |
mo,
|
271 |
+
package_analysis_stack,
|
272 |
+
package_meta,
|
273 |
purge_tabs,
|
274 |
sc_m,
|
275 |
schema_editors,
|
276 |
selection_table,
|
277 |
+
ss_asset_details,
|
278 |
upload_func,
|
279 |
+
yaml_template,
|
280 |
):
|
281 |
+
client_section = mo.md(f'''
|
282 |
###**Instantiate your watsonx.ai client:**
|
283 |
|
284 |
1. Select a region from the dropdown menu
|
|
|
291 |
|
292 |
---
|
293 |
|
294 |
+
{client_stack}
|
|
|
|
|
|
|
|
|
295 |
|
296 |
''')
|
297 |
|
|
|
304 |
}
|
305 |
)
|
306 |
|
307 |
+
function_section = mo.md(f'''###**Create your function from the template:**
|
308 |
|
309 |
1. Use the code editor window to create a function to deploy
|
310 |
<br>
|
|
|
343 |
|
344 |
''')
|
345 |
|
346 |
+
upload_section = mo.md(f'''
|
347 |
###**Review and Upload your function**
|
348 |
|
349 |
1. Review the function metadata specs JSON
|
|
|
364 |
|
365 |
''')
|
366 |
|
367 |
+
deployment_section = mo.md(f'''
|
368 |
###**Deploy your function:**
|
369 |
|
370 |
1. Select a hardware specification (vCPUs/GB) that you want your function deployed on
|
|
|
397 |
|
398 |
''')
|
399 |
|
400 |
+
purging_section = mo.md(f'''
|
401 |
###**Helper Purge Functions:**
|
402 |
+
|
403 |
These functions help you retrieve, select and delete deployments, data assets or repository assets (functions, models, etc.) that you have in the deployment space. This is meant to support fast cleanup.
|
404 |
+
|
405 |
Select the tab based on what you want to delete, then click each of the buttons one by one after the previous gives a response.
|
406 |
+
|
407 |
---
|
408 |
|
409 |
{purge_tabs}
|
410 |
|
411 |
''')
|
412 |
|
413 |
+
|
414 |
+
packages_section = mo.md(f'''
|
415 |
+
###**If needed - Create a custom software-spec with added python packages**
|
416 |
+
|
417 |
+
1. Check to see if the python library you want to use is already available inside watsonx.ai's runtime environment base specs for deployed functions by adding them as a comma separated list, e.g. - plotly, ibm-watsonx-ai==1.3.6, etc. into the text area.
|
418 |
+
|
419 |
+
2. If you wish to see all the packages already present, check the box to return it alognside your validation results.
|
420 |
+
|
421 |
+
---
|
422 |
+
|
423 |
+
{package_analysis_stack}
|
424 |
+
|
425 |
+
---
|
426 |
+
|
427 |
+
3. If it isn't, you can create a custom software spec that adds a package_extension add-on to it. Use the template selector to see some examples, but when you are ready add your packages in the code editor after the "pip:" section.
|
428 |
+
|
429 |
+
{yaml_template}
|
430 |
+
|
431 |
+
4. Give your package extension and software spec names and descriptions and submit.
|
432 |
+
|
433 |
+
5. You will find it in the Function Upload table afterward.
|
434 |
+
|
435 |
+
---
|
436 |
+
|
437 |
+
{package_meta}
|
438 |
+
|
439 |
+
---
|
440 |
+
|
441 |
+
Results:
|
442 |
+
|
443 |
+
{ss_asset_details}
|
444 |
+
|
445 |
+
''')
|
446 |
+
return (
|
447 |
+
client_section,
|
448 |
+
deployment_section,
|
449 |
+
function_section,
|
450 |
+
packages_section,
|
451 |
+
upload_section,
|
452 |
)
|
453 |
|
454 |
+
|
455 |
+
@app.cell
|
456 |
+
def _(client_section, mo):
|
457 |
+
ui_accordion_section_1 = mo.accordion(
|
458 |
+
{"Section 1: **watsonx.ai Credentials**": client_section}
|
459 |
+
)
|
460 |
+
ui_accordion_section_1
|
461 |
+
return
|
462 |
+
|
463 |
+
|
464 |
+
@app.cell
|
465 |
+
def _(function_section, mo):
|
466 |
+
ui_accordion_section_2 = mo.accordion(
|
467 |
+
{"Section 2: **Function Creation**": function_section}
|
468 |
+
)
|
469 |
+
ui_accordion_section_2
|
470 |
+
return
|
471 |
+
|
472 |
+
|
473 |
+
@app.cell
|
474 |
+
def _(mo, packages_section):
|
475 |
+
ui_accordion_section_3 = mo.accordion(
|
476 |
+
{"Section 3: **Create a Package Extension (Optional)**": packages_section}
|
477 |
+
)
|
478 |
+
ui_accordion_section_3
|
479 |
+
return
|
480 |
+
|
481 |
+
|
482 |
+
@app.cell
|
483 |
+
def _(mo, upload_section):
|
484 |
+
ui_accordion_section_4 = mo.accordion(
|
485 |
+
{"Section 4: **Function Upload**": upload_section}
|
486 |
+
)
|
487 |
+
ui_accordion_section_4
|
488 |
+
return
|
489 |
+
|
490 |
+
|
491 |
+
@app.cell
|
492 |
+
def _(deployment_section, mo):
|
493 |
+
ui_accordion_section_5 = mo.accordion(
|
494 |
+
{"Section 5: **Function Deployment**": deployment_section}
|
495 |
+
)
|
496 |
+
ui_accordion_section_5
|
497 |
+
return
|
498 |
+
|
499 |
+
|
500 |
+
@app.cell
|
501 |
+
def _(mo, packages_section):
|
502 |
+
ui_accordion_section_6 = mo.accordion(
|
503 |
+
{"Section 6: **Helper Functions**": packages_section}
|
504 |
+
)
|
505 |
+
ui_accordion_section_6
|
506 |
+
return
|
507 |
|
508 |
|
509 |
@app.cell
|
|
|
533 |
|
534 |
return score
|
535 |
|
536 |
+
score = your_function_name()
|
537 |
'''
|
538 |
|
539 |
function_editor = (
|
|
|
544 |
|
545 |
''')
|
546 |
.batch(
|
547 |
+
editor = mo.ui.code_editor(value=template, language="python", min_height=200, show_copy_button=True, theme="dark")
|
548 |
)
|
549 |
.form(show_clear_button=True, bordered=False)
|
550 |
)
|
551 |
|
552 |
# function_editor
|
553 |
+
return (function_editor,)
|
554 |
|
555 |
|
556 |
@app.cell
|
557 |
+
def _(ast, function_editor, mo, os):
|
558 |
function_name = None
|
559 |
if function_editor.value:
|
560 |
# Get the edited code from the function editor
|
|
|
568 |
if isinstance(node, ast.FunctionDef):
|
569 |
function_name = node.name
|
570 |
break
|
571 |
+
|
572 |
if function_name is not None:
|
573 |
# Set deployable_function to None since we're not executing the code
|
574 |
deployable_function = None
|
575 |
mo.md(f"Found function: '{function_name}' (using file path approach)")
|
576 |
+
|
577 |
# Create the directory if it doesn't exist
|
578 |
save_dir = "/tmp/notebook_functions"
|
579 |
os.makedirs(save_dir, exist_ok=True)
|
|
|
585 |
mo.md("No function found in the editor code")
|
586 |
except SyntaxError as e:
|
587 |
mo.md(f"Syntax error in function code: {str(e)}")
|
588 |
+
return (function_name,)
|
589 |
+
|
590 |
+
|
591 |
+
@app.cell
|
592 |
+
def _():
|
593 |
+
yaml_templates = {
|
594 |
+
"empty": """dependencies:
|
595 |
+
- pip
|
596 |
+
- pip:
|
597 |
+
- <library_name>
|
598 |
+
- ...
|
599 |
+
""",
|
600 |
+
"llama_index_and_scikit": """dependencies:
|
601 |
+
- pip
|
602 |
+
- pip:
|
603 |
+
- scikit-learn==1.6.1
|
604 |
+
- scikit-llm==1.4.1
|
605 |
+
- plotly==6.0.1
|
606 |
+
- altair==5.5.0
|
607 |
+
- PyMuPDF==1.25.1
|
608 |
+
- llama-index==0.12.24
|
609 |
+
- llama-index-readers-file==0.4.6
|
610 |
+
- llama-index-readers-web==0.3.8
|
611 |
+
""",
|
612 |
+
"data_product_hub": """dependencies:
|
613 |
+
- pip
|
614 |
+
- pip:
|
615 |
+
- PyMuPDF==1.25.1
|
616 |
+
- llama-index==0.12.24
|
617 |
+
- llama-index-readers-file==0.4.6
|
618 |
+
- llama-index-readers-web==0.3.8
|
619 |
+
- plotly==6.0.1
|
620 |
+
- altair==5.5.0
|
621 |
+
- dask==2025.2.0
|
622 |
+
- dph-services==0.4.0
|
623 |
+
""",
|
624 |
+
"watsonx_data": """dependencies:
|
625 |
+
- pip
|
626 |
+
- pip:
|
627 |
+
- prestodb
|
628 |
+
- PyMuPDF==1.25.1
|
629 |
+
- llama-index==0.12.24
|
630 |
+
- llama-index-readers-file==0.4.6
|
631 |
+
- llama-index-readers-web==0.3.8
|
632 |
+
- ibm-watsonxdata==0.4.0
|
633 |
+
"""
|
634 |
+
}
|
635 |
+
return (yaml_templates,)
|
636 |
+
|
637 |
+
|
638 |
+
@app.cell
|
639 |
+
def _(mo, yaml_templates):
|
640 |
+
yaml_template = mo.ui.dropdown(yaml_templates, searchable=True, label="**Select a template:**", value="empty")
|
641 |
+
return (yaml_template,)
|
642 |
+
|
643 |
+
|
644 |
+
@app.cell
|
645 |
+
def _(tempfile):
|
646 |
+
def create_yaml_tempfile(yaml_editor_value):
|
647 |
+
"""Creates temporary YAML file and returns its path"""
|
648 |
+
temp_file = tempfile.NamedTemporaryFile(suffix='.yaml', delete=False)
|
649 |
+
|
650 |
+
# Access the actual YAML content within the dictionary structure
|
651 |
+
if isinstance(yaml_editor_value, dict) and 'yml_editor' in yaml_editor_value:
|
652 |
+
# Extract the YAML content string from the yml_editor key
|
653 |
+
yaml_content = yaml_editor_value['yml_editor']
|
654 |
+
else:
|
655 |
+
# Use as is if it's already a string or has unexpected structure
|
656 |
+
yaml_content = yaml_editor_value
|
657 |
+
|
658 |
+
# Write the content to the file
|
659 |
+
with open(temp_file.name, 'w') as f:
|
660 |
+
f.write(str(yaml_content))
|
661 |
+
|
662 |
+
return temp_file.name
|
663 |
+
return (create_yaml_tempfile,)
|
664 |
+
|
665 |
+
|
666 |
+
@app.cell
|
667 |
+
def _(mo, yaml_template):
|
668 |
+
pkg_types = {"Conda Yaml":"conda_yml","Custom user library":"custom_library"}
|
669 |
+
|
670 |
+
package_meta = (
|
671 |
+
mo.md('''**Create your Conda YAML by editing the template:**
|
672 |
+
|
673 |
+
{yml_editor}
|
674 |
+
|
675 |
+
{package_name}
|
676 |
+
|
677 |
+
{package_description}
|
678 |
+
|
679 |
+
{software_spec_name}
|
680 |
+
|
681 |
+
{software_spec_description}
|
682 |
+
''')
|
683 |
+
.batch(
|
684 |
+
yml_editor = mo.ui.code_editor(value=yaml_template.value, language="yaml", min_height=100, theme="dark"),
|
685 |
+
package_name = mo.ui.text(placeholder="Python Package for...",
|
686 |
+
label="Package Extension Name:",
|
687 |
+
kind="text",
|
688 |
+
value="Custom Python Package"),
|
689 |
+
software_spec_name = mo.ui.text(placeholder="Software Spec Name",
|
690 |
+
label="Custom Software Spec Name:",
|
691 |
+
kind="text", value="Extended Python Function Software Spec"),
|
692 |
+
package_description = mo.ui.text_area(placeholder="Write a description for your package.",
|
693 |
+
label="Package Description:",
|
694 |
+
value=" "),
|
695 |
+
software_spec_description = mo.ui.text_area(placeholder="Write a description for your software spec.",
|
696 |
+
label="Software Spec Description:",
|
697 |
+
value=" "),
|
698 |
+
package_type = mo.ui.dropdown(pkg_types,
|
699 |
+
label="Select your package type:",
|
700 |
+
value="Conda Yaml")
|
701 |
+
)
|
702 |
+
.form(show_clear_button=True, bordered=False)
|
703 |
+
)
|
704 |
+
return (package_meta,)
|
705 |
+
|
706 |
+
|
707 |
+
@app.cell
|
708 |
+
def _(mo):
|
709 |
+
check_packages =(mo.md('''
|
710 |
+
**Check if a package you want to use is in the base software_specification already:**
|
711 |
+
|
712 |
+
{package_list}
|
713 |
+
|
714 |
+
{return_full_list}
|
715 |
+
''')
|
716 |
+
.batch(
|
717 |
+
package_list = mo.ui.text_area(placeholder="Add packages as a comma separated list (with or without versions)."),
|
718 |
+
return_full_list = mo.ui.checkbox(value=False, label="Return a full list of packages in the base software specification.")
|
719 |
+
)
|
720 |
+
.form(show_clear_button=True, bordered=False)
|
721 |
)
|
722 |
+
return (check_packages,)
|
723 |
+
|
724 |
+
|
725 |
+
@app.cell
|
726 |
+
def _(check_packages):
|
727 |
+
if check_packages.value is not None:
|
728 |
+
packages = check_packages.value['package_list']
|
729 |
+
verification_list = [item.strip() for item in packages.split(',')]
|
730 |
+
full_list_return = check_packages.value['return_full_list']
|
731 |
+
else:
|
732 |
+
packages = None
|
733 |
+
verification_list = None
|
734 |
+
full_list_return = None
|
735 |
+
return full_list_return, verification_list
|
736 |
+
|
737 |
+
|
738 |
+
@app.cell
|
739 |
+
def _(
|
740 |
+
analyze_software_spec,
|
741 |
+
base_software_spec,
|
742 |
+
full_list_return,
|
743 |
+
verification_list,
|
744 |
+
visualize_software_spec,
|
745 |
+
):
|
746 |
+
if verification_list is not None:
|
747 |
+
pkg_analysis = analyze_software_spec(base_software_spec, verification_list, return_full_sw_package_list=full_list_return)
|
748 |
+
package_df = visualize_software_spec(pkg_analysis, verification_list)
|
749 |
+
else:
|
750 |
+
pkg_analysis = None
|
751 |
+
package_df = None
|
752 |
+
return (package_df,)
|
753 |
+
|
754 |
+
|
755 |
+
@app.cell
|
756 |
+
def _(check_packages, mo, package_df):
|
757 |
+
package_analysis_stack = mo.vstack([check_packages, package_df], justify="center")
|
758 |
+
return (package_analysis_stack,)
|
759 |
+
|
760 |
+
|
761 |
+
@app.cell
|
762 |
+
def _(deployment_client):
|
763 |
+
if deployment_client is not None:
|
764 |
+
base_sw_spec_id = "45f12dfe-aa78-5b8d-9f38-0ee223c47309"
|
765 |
+
base_software_spec = deployment_client.software_specifications.get_details(base_sw_spec_id)
|
766 |
+
else:
|
767 |
+
base_sw_spec_id = None
|
768 |
+
base_software_spec = None
|
769 |
+
return base_software_spec, base_sw_spec_id
|
770 |
+
|
771 |
+
|
772 |
+
@app.cell
|
773 |
+
def _(create_yaml_tempfile, deployment_client, package_meta):
|
774 |
+
if package_meta.value is not None and deployment_client is not None:
|
775 |
+
pe_metadata = {
|
776 |
+
deployment_client.package_extensions.ConfigurationMetaNames.NAME: package_meta.value['package_name'],
|
777 |
+
deployment_client.package_extensions.ConfigurationMetaNames.TYPE: package_meta.value['package_type'],
|
778 |
+
deployment_client.software_specifications.ConfigurationMetaNames.DESCRIPTION:package_meta.value['package_description']
|
779 |
+
}
|
780 |
+
yaml_file_path = create_yaml_tempfile(package_meta.value['yml_editor'])
|
781 |
+
else:
|
782 |
+
pe_metadata = {
|
783 |
+
}
|
784 |
+
yaml_file_path = None
|
785 |
+
|
786 |
+
return pe_metadata, yaml_file_path
|
787 |
+
|
788 |
+
|
789 |
+
@app.cell
|
790 |
+
def _(
|
791 |
+
base_sw_spec_id,
|
792 |
+
deployment_client,
|
793 |
+
package_meta,
|
794 |
+
pe_metadata,
|
795 |
+
yaml_file_path,
|
796 |
+
):
|
797 |
+
if yaml_file_path is not None:
|
798 |
+
### Stores the package extension
|
799 |
+
pe_asset_details = deployment_client.package_extensions.store(
|
800 |
+
meta_props=pe_metadata,
|
801 |
+
file_path=yaml_file_path
|
802 |
+
)
|
803 |
+
package_id = pe_asset_details["metadata"]["asset_id"]
|
804 |
+
### Creates a custom software specification based on the standard python function spec_id - "45f12dfe-aa78-5b8d-9f38-0ee223c47309"
|
805 |
+
ss_metadata = {
|
806 |
+
deployment_client.software_specifications.ConfigurationMetaNames.NAME: package_meta.value['software_spec_name'],
|
807 |
+
deployment_client.software_specifications.ConfigurationMetaNames.DESCRIPTION: package_meta.value['software_spec_description'],
|
808 |
+
deployment_client.software_specifications.ConfigurationMetaNames.BASE_SOFTWARE_SPECIFICATION: {'guid': base_sw_spec_id},
|
809 |
+
deployment_client.software_specifications.ConfigurationMetaNames.PACKAGE_EXTENSIONS: [{'guid': package_id}]
|
810 |
+
}
|
811 |
+
ss_asset_details = deployment_client.software_specifications.store(meta_props=ss_metadata)
|
812 |
+
else:
|
813 |
+
pe_asset_details = None
|
814 |
+
package_id = None
|
815 |
+
ss_metadata = {}
|
816 |
+
ss_asset_details = None
|
817 |
+
|
818 |
+
# ss_asset_details
|
819 |
+
return (ss_asset_details,)
|
820 |
|
821 |
|
822 |
@app.cell
|
|
|
897 |
initial_selection=[0]
|
898 |
)
|
899 |
|
900 |
+
return (selection_table,)
|
|
|
|
|
|
|
|
|
|
|
|
|
901 |
|
902 |
|
903 |
@app.cell
|
904 |
def _(mo):
|
905 |
input_schema_checkbox = mo.ui.checkbox(label="Add input schema (optional)")
|
906 |
output_schema_checkbox = mo.ui.checkbox(label="Add output schema (optional)")
|
907 |
+
# sample_input_checkbox = mo.ui.checkbox(label="Add sample input example (optional)")
|
908 |
+
return input_schema_checkbox, output_schema_checkbox
|
909 |
|
910 |
|
911 |
@app.cell
|
912 |
def _(
|
913 |
+
function_name,
|
914 |
input_schema_checkbox,
|
915 |
mo,
|
916 |
output_schema_checkbox,
|
|
|
917 |
selection_table,
|
918 |
template_variant,
|
919 |
):
|
|
|
957 |
schema_metadata=mo.hstack([
|
958 |
input_schema_checkbox,
|
959 |
output_schema_checkbox,
|
960 |
+
# sample_input_checkbox
|
961 |
],
|
962 |
justify="center", gap=1, align="center", wrap=True
|
963 |
)
|
|
|
978 |
return (
|
979 |
description_input,
|
980 |
fm,
|
|
|
|
|
981 |
sc_m,
|
|
|
982 |
software_spec,
|
983 |
tags_editor,
|
984 |
uploaded_function_name,
|
|
|
988 |
@app.cell
|
989 |
def _(json, mo, template_variant):
|
990 |
if template_variant.value == "Stream Files to IBM COS [Example]":
|
991 |
+
from cos_stream_schema_examples import input_schema, output_schema
|
992 |
else:
|
993 |
input_schema = [
|
994 |
{
|
|
|
1026 |
}
|
1027 |
]
|
1028 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1029 |
|
1030 |
|
1031 |
+
input_schema_editor = mo.ui.code_editor(value=json.dumps(input_schema, indent=4), language="python", min_height=100,theme="dark")
|
1032 |
+
output_schema_editor = mo.ui.code_editor(value=json.dumps(output_schema, indent=4), language="python", min_height=100,theme="dark")
|
|
|
1033 |
|
1034 |
schema_editors = mo.accordion(
|
1035 |
{
|
1036 |
"""**Input Schema Metadata Editor**""": input_schema_editor,
|
1037 |
"""**Output Schema Metadata Editor**""": output_schema_editor,
|
|
|
1038 |
}, multiple=True
|
1039 |
)
|
1040 |
|
1041 |
# schema_editors
|
1042 |
+
return input_schema_editor, output_schema_editor, schema_editors
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1043 |
|
1044 |
|
1045 |
@app.cell
|
|
|
1055 |
os,
|
1056 |
output_schema_checkbox,
|
1057 |
output_schema_editor,
|
|
|
|
|
1058 |
selection_table,
|
1059 |
software_spec,
|
1060 |
tags_editor,
|
|
|
1099 |
function_meta[deployment_client.repository.FunctionMetaNames.OUTPUT_DATA_SCHEMAS] = ast.literal_eval(output_schema_editor.value)
|
1100 |
|
1101 |
# Add sample input if checkbox is checked
|
1102 |
+
# if sample_input_checkbox.value:
|
1103 |
+
# try:
|
1104 |
+
# function_meta[deployment_client.repository.FunctionMetaNames.SAMPLE_SCORING_INPUT] = json.loads(sample_input_editor.value)
|
1105 |
+
# except json.JSONDecodeError:
|
1106 |
+
# # If JSON parsing fails, try Python literal evaluation as fallback
|
1107 |
+
# function_meta[deployment_client.repository.FunctionMetaNames.SAMPLE_SCORING_INPUT] = ast.literal_eval(sample_input_editor.value)
|
1108 |
|
1109 |
def upload_function(function_meta, use_function_object=False):
|
1110 |
"""
|
|
|
1158 |
# # Upload using the absolute file path
|
1159 |
# abs_file_path = os.path.abspath(file_path)
|
1160 |
# mo.md(f"Uploading function from file: {abs_file_path}")
|
1161 |
+
|
1162 |
# # Using the absolute file path might help in some environments
|
1163 |
# func_details = deployment_client.repository.store_function(abs_file_path, function_meta)
|
1164 |
# Upload using the file path approach
|
|
|
1166 |
# Create a zip file of the Python module
|
1167 |
import gzip
|
1168 |
import shutil
|
1169 |
+
|
1170 |
# Path for the gzipped file
|
1171 |
gz_path = f"{save_dir}/{func_name}.py.gz"
|
1172 |
+
|
1173 |
# Create gzip file
|
1174 |
with open(file_path, 'rb') as f_in:
|
1175 |
with gzip.open(gz_path, 'wb') as f_out:
|
1176 |
shutil.copyfileobj(f_in, f_out)
|
1177 |
+
|
1178 |
# Upload using the gzipped file path
|
1179 |
mo.md(f"Uploading function from gzip: {gz_path}")
|
1180 |
func_details = deployment_client.repository.store_function(gz_path, function_meta)
|
|
|
1202 |
)
|
1203 |
|
1204 |
# function_meta
|
1205 |
+
return get_upload_status, upload_button
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1206 |
|
1207 |
|
1208 |
@app.cell
|
|
|
1219 |
upload_button,
|
1220 |
mo.md(f"**Status:** {get_upload_status()}")
|
1221 |
], justify="space-around", align="center")
|
1222 |
+
return artifact_id, upload_func
|
1223 |
|
1224 |
|
1225 |
@app.cell
|
|
|
1302 |
initial_selection=[0]
|
1303 |
)
|
1304 |
|
1305 |
+
return deployment_name, deployment_type, hw_selection_table
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1306 |
|
1307 |
|
1308 |
@app.cell
|
|
|
1314 |
deployment_type,
|
1315 |
hw_selection_table,
|
1316 |
mo,
|
|
|
1317 |
upload_button,
|
1318 |
):
|
1319 |
def deploy_function(artifact_id, deployment_type):
|
|
|
1392 |
], justify="space-around")
|
1393 |
|
1394 |
# deployment_definition
|
1395 |
+
return deploy_button, deployment_definition
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1396 |
|
1397 |
|
1398 |
@app.cell
|
|
|
1408 |
|
1409 |
|
1410 |
@app.cell(hide_code=True)
|
1411 |
+
def _(deployment_client, mo, pd):
|
1412 |
### Functions to List , Get ID's as a list and Purge of Assets
|
1413 |
|
1414 |
def get_deployment_list():
|
|
|
1443 |
return repository_list
|
1444 |
|
1445 |
#----
|
1446 |
+
|
1447 |
def delete_with_progress(ids_list, delete_function, item_type="items"):
|
1448 |
"""
|
1449 |
Generic wrapper that adds a progress bar to any deletion function
|
|
|
1490 |
delete_data_assets,
|
1491 |
delete_deployments,
|
1492 |
delete_repository_items,
|
|
|
1493 |
get_data_asset_ids,
|
1494 |
get_data_assets_list,
|
1495 |
get_deployment_ids,
|
|
|
1498 |
get_repository_list,
|
1499 |
)
|
1500 |
|
1501 |
+
|
1502 |
@app.cell
|
1503 |
def _(get_data_assets_tab, get_deployments_tab, get_repository_tab, mo):
|
1504 |
if get_deployments_tab() is not None:
|
|
|
1515 |
data_assets_table = mo.ui.table(get_data_assets_tab())
|
1516 |
else:
|
1517 |
data_assets_table = mo.md("No Table Loaded")
|
1518 |
+
|
1519 |
return data_assets_table, deployments_table, repository_table
|
1520 |
|
1521 |
+
|
1522 |
@app.cell
|
1523 |
+
def _(
|
1524 |
+
deployments_table,
|
1525 |
+
get_deployment_id_list,
|
1526 |
+
get_deployments_button,
|
1527 |
+
mo,
|
1528 |
+
purge_deployments,
|
1529 |
+
):
|
1530 |
deployments_purge_stack = mo.hstack([get_deployments_button, get_deployment_id_list, purge_deployments])
|
1531 |
deployments_purge_stack_results = mo.vstack([deployments_table, get_deployment_id_list.value, purge_deployments.value])
|
1532 |
|
1533 |
deployments_purge_tab = mo.vstack([deployments_purge_stack, deployments_purge_stack_results])
|
1534 |
+
return (deployments_purge_tab,)
|
|
|
|
|
|
|
|
|
1535 |
|
1536 |
|
1537 |
@app.cell
|
1538 |
+
def _(
|
1539 |
+
get_repository_button,
|
1540 |
+
get_repository_id_list,
|
1541 |
+
mo,
|
1542 |
+
purge_repository,
|
1543 |
+
repository_table,
|
1544 |
+
):
|
1545 |
repository_purge_stack = mo.hstack([get_repository_button, get_repository_id_list, purge_repository])
|
1546 |
|
1547 |
repository_purge_stack_results = mo.vstack([repository_table, get_repository_id_list.value, purge_repository.value])
|
1548 |
|
1549 |
repository_purge_tab = mo.vstack([repository_purge_stack, repository_purge_stack_results])
|
1550 |
+
return (repository_purge_tab,)
|
|
|
|
|
|
|
|
|
1551 |
|
1552 |
|
1553 |
@app.cell
|
1554 |
+
def _(
|
1555 |
+
data_assets_table,
|
1556 |
+
get_data_asset_id_list,
|
1557 |
+
get_data_assets_button,
|
1558 |
+
mo,
|
1559 |
+
purge_data_assets,
|
1560 |
+
):
|
1561 |
data_assets_purge_stack = mo.hstack([get_data_assets_button, get_data_asset_id_list, purge_data_assets])
|
1562 |
data_assets_purge_stack_results = mo.vstack([data_assets_table, get_data_asset_id_list.value, purge_data_assets.value])
|
1563 |
|
1564 |
data_assets_purge_tab = mo.vstack([data_assets_purge_stack, data_assets_purge_stack_results])
|
1565 |
+
return (data_assets_purge_tab,)
|
|
|
|
|
|
|
|
|
1566 |
|
1567 |
|
1568 |
@app.cell
|
|
|
1571 |
{"Purge Deployments": deployments_purge_tab, "Purge Repository Assets": repository_purge_tab,"Purge Data Assets": data_assets_purge_tab }, lazy=False
|
1572 |
)
|
1573 |
|
|
|
1574 |
return (purge_tabs,)
|
1575 |
|
1576 |
+
|
1577 |
@app.cell
|
1578 |
def _(mo):
|
1579 |
get_deployments_tab, set_deployments_tab = mo.state(None)
|