from nodes.scraping.html import * from nodes.llm.textmodel import ai_response as generate_ai_response from nodes.socialmedia.x import * from nodes.socialmedia.reddit import reddit_post from nodes.processing.list import * #updating the executor to process multiple nodes for better execution (Mon jan 6 2025) def execute(workflow): print(workflow) # Nodes that do not depend on any other node for input unique_node = [] # Nodes that depend on other nodes for input dependant_node = [] # Temporary storage for saving the output of the nodes temp_data = [] # Assigning unique or dependent nodes to their respective array for step in workflow['steps']: # Check if 'config' contains a reference to other nodes is_dependant = False if 'config' in step and isinstance(step['config'], dict): for key, value in step['config'].items(): # If the value contains a reference like 'node-1', 'node-2', etc. if isinstance(value, str) and value.startswith('node-'): is_dependant = True break if is_dependant: dependant_node.append(step) else: unique_node.append(step) for step in workflow['steps']: print("executing step", step['node'], step['type']) if step['type'] == "scrape_html": for temp in temp_data: if temp['node'] == step['config']['url']: print(temp['data']) data = scrape_html(temp['data']) temp_data.append({'node': step['node'], 'data': data, 'type':step['type']}) break else: data = scrape_html(step['config']['url']) temp_data.append({'node': step['node'], 'data': data, 'type':step['type']}) elif step['type'] == "scrape_images": for temp in temp_data: if temp['node'] == step['config']['data']: filter = step['config']['filter'] data = scrape_images(data=temp['data'], filter=filter) temp_data.append({'node': step['node'], 'data': data, 'type':step['type']}) break else: data = step['config']['data'] filter = step['config']['filter'] response = scrape_images(data=data, filter=filter) temp_data.append({'node': step['node'], 'data': response, 'type':step['type']}) elif step['type'] == "scrape_links": for temp in temp_data: if temp['node'] == step['config']['data']: data = scrape_links(url = temp['data'], filter=step['config']['filter']) temp_data.append({'node': step['node'], 'data': data, 'type':step['type']}) break else: data = scrape_links(url = step['config']['data'], filter=step['config']['filter']) temp_data.append({'node': step['node'], 'data': data, 'type':step['type']}) elif step['type'] == "scrape_metadata": for temp in temp_data: if temp['node'] == step['config']['data']: data = scrape_text(temp['data']) temp_data.append({'node': step['node'], 'data': data, 'type':step['type']}) break else: data = step['config']['data'] scrape_text(data) temp_data.append({'node': step['node'], 'data': data, 'type':step['type']}) elif step['type'] == "scrape_text": for temp in temp_data: if temp['node'] == step['config']['data']: data = scrape_text(str(temp['data'])) temp_data.append({'node': step['node'], 'data': data, 'type':step['type']}) break else: data = step['config']['data'] scrape_text(data) temp_data.append({'node': step['node'], 'data': data, 'type':step['type']}) elif step['type'] == "create_tweet": for temp in temp_data: print("node", temp['node']) if temp['node'] == step['config']['data']: print('foudntemp node') login = step['config']['login'] data = create_tweet(text=temp['data'], login=login) temp_data.append({'node': step['node'], 'data': data, 'type':step['type']}) break else: print('not found node') data = step['config']['data'] login = step['config']['login'] data = create_tweet(text=data, login=login) temp_data.append({'node': step['node'], 'data': data, 'type':step['type']}) elif step['type'] == "create_reddit": for temp in temp_data: print("node", temp['node']) if temp['node'] == step['config']['data']: print('foud temp node') username = step['config']['username'] password = step['config']['password'] subreddit = step['config']['subreddit'] client_id = step['config']['id'] client_secret = step['config']['secret'] title = step['config']['secret'] data = reddit_post(client_id, client_secret, username, password, subreddit, title, body=temp['data']) temp_data.append({'node': step['node'], 'data': data, 'type':step['type']}) break else: username = step['config']['username'] password = step['config']['password'] subreddit = step['config']['subreddit'] client_id = step['config']['id'] client_secret = step['config']['secret'] title = step['config']['secret'] data = reddit_post(client_id, client_secret, username, password, subreddit, title, body=step['config']['data']) temp_data.append({'node': step['node'], 'data': data, 'type':step['type']}) elif step["type"] == "ai_response": for temp in temp_data: if temp['node'] == step['config']['data']: data = generate_ai_response(step['config']['prompt'] + str(temp['data'])) temp_data.append({'node': step['node'], 'data': data, 'type':step['type']}) break else: data = generate_ai_response(prompt=step['config']['prompt']) temp_data.append({'node': step['node'], 'data': data, 'type':step['type']}) elif step['type'] == "scrape_div": for temp in temp_data: if temp['node'] == step['config']['data']: data = scrape_div(data=temp['data'], div=step['config']['class']) temp_data.append({'node': step['node'], 'data': data, 'type':step['type']}) break else: data = step['config']['data'] class_ = step['config']['class'] data = scrape_div(data=data, div=class_) temp_data.append({'node': step['node'], 'data': data, 'type':step['type']}) elif step['type'] == "extract_element": for temp in temp_data: if temp['node'] == step['config']['data']: print(step['config']['data']) data = extract_element(list_=temp['data'], index=step['config']['index'], value=step['config']['value']) temp_data.append({'node': step['node'], 'data': data, 'type':step['type']}) break else: data = extract_element(list_=step['config']['data'], index=step['config']['index'], value=step['config']['value']) temp_data.append({'node': step['node'], 'data': data, 'type':step['type']}) elif step['type'] == "create_list": data = step['config']['data'] # Convert string representation of list to actual list if isinstance(data, str): try: data = eval(data) except: pass temp_data.append({'node': step['node'], 'data': data, 'type':step['type']}) elif step['type'] == "reddit_post": title = step['config']['title'] content = step['config']['data'] # Check if title is from another node for temp in temp_data: if temp['node'] == title: title = temp['data'] break # Check if content is from another node for temp in temp_data: if temp['node'] == content: content = temp['data'] break data = reddit_post( client_id=step['config']['client_id'], client_secret=step['config']['client_secret'], username=step['config']['username'], password=step['config']['password'], subreddit=step['config']['subreddit'], title=title, body=content ) temp_data.append({'node': step['node'], 'data': data, 'type': step['type']}) return temp_data