added gradio chat bot interface

.gitignore

@@ -1 +1,2 @@
-.env
+.env
+*.pyc

data/data.txt

@@ -1,45 +0,0 @@
-LOCAL_POSITION_NED {time_boot_ms : 239230, x : 109.40231323242188, y : 37.670654296875, z : -9.969955444335938, vx : -0.018984168767929077, vy : -0.005424499046057463, vz : 5.3835941798752174e-05}
-LOCAL_POSITION_NED {time_boot_ms : 239480, x : 109.40264129638672, y : 37.67082977294922, z : -9.96972942352295, vx : -0.019311394542455673, vy : -0.005466990172863007, vz : -5.457072620629333e-05}
-LOCAL_POSITION_NED {time_boot_ms : 239729, x : 109.4027099609375, y : 37.67092514038086, z : -9.969511032104492, vx : -0.01970984973013401, vy : -0.005530376452952623, vz : -0.0002979390264954418}
-LOCAL_POSITION_NED {time_boot_ms : 239979, x : 109.40276336669922, y : 37.671024322509766, z : -9.969305992126465, vx : -0.020000549033284187, vy : -0.0055604190565645695, vz : -0.0005820284713990986}
-LOCAL_POSITION_NED {time_boot_ms : 240229, x : 109.40313720703125, y : 37.67121887207031, z : -9.969114303588867, vx : -0.02006402052938938, vy : -0.00547247938811779, vz : -0.000677842297591269}
-LOCAL_POSITION_NED {time_boot_ms : 240479, x : 109.4032211303711, y : 37.67136764526367, z : -9.968948364257812, vx : -0.0202533807605505, vy : -0.005525950342416763, vz : -0.0007673363434150815}
-LOCAL_POSITION_NED {time_boot_ms : 240729, x : 109.40300750732422, y : 37.67150115966797, z : -9.968831062316895, vx : -0.020623182877898216, vy : -0.0055752284824848175, vz : -0.0009132251143455505}
-LOCAL_POSITION_NED {time_boot_ms : 240980, x : 109.4027328491211, y : 37.67162322998047, z : -9.968841552734375, vx : -0.021015815436840057, vy : -0.0057647754438221455, vz : -0.0011039149248972535}
-LOCAL_POSITION_NED {time_boot_ms : 241230, x : 109.40284729003906, y : 37.67182540893555, z : -9.96899127960205, vx : -0.020866867154836655, vy : -0.005960384849458933, vz : -0.0008990211645141244}
-LOCAL_POSITION_NED {time_boot_ms : 241480, x : 109.40279388427734, y : 37.67191696166992, z : -9.96912670135498, vx : -0.020786112174391747, vy : -0.006111173890531063, vz : -0.000912059098482132}
-LOCAL_POSITION_NED {time_boot_ms : 241729, x : 109.40264129638672, y : 37.67238998413086, z : -9.969264030456543, vx : -0.020849552005529404, vy : -0.006145048886537552, vz : -0.0006911428063176572}
-LOCAL_POSITION_NED {time_boot_ms : 241979, x : 109.40258026123047, y : 37.673919677734375, z : -9.969388008117676, vx : -0.021021265536546707, vy : -0.006493883207440376, vz : -0.0007432479178532958}
-LOCAL_POSITION_NED {time_boot_ms : 242229, x : 109.40287017822266, y : 37.675846099853516, z : -9.96950626373291, vx : -0.021021228283643723, vy : -0.007321516517549753, vz : -0.0005821734084747732}
-LOCAL_POSITION_NED {time_boot_ms : 242479, x : 109.40294647216797, y : 37.67726135253906, z : -9.969613075256348, vx : -0.02099592424929142, vy : -0.008481874130666256, vz : -0.000457199988886714}
-LOCAL_POSITION_NED {time_boot_ms : 242729, x : 109.40283203125, y : 37.67806625366211, z : -9.969687461853027, vx : -0.02108212560415268, vy : -0.009890645742416382, vz : -0.000333481642883271}
-LOCAL_POSITION_NED {time_boot_ms : 242979, x : 109.40362548828125, y : 37.67841339111328, z : -9.96973705291748, vx : -0.02084990404546261, vy : -0.011260980740189552, vz : -0.00045963365118950605}
-LOCAL_POSITION_NED {time_boot_ms : 243230, x : 109.40609741210938, y : 37.678443908691406, z : -9.969796180725098, vx : -0.021079031750559807, vy : -0.012367655523121357, vz : -0.0001387261290801689}
-LOCAL_POSITION_NED {time_boot_ms : 243480, x : 109.40825653076172, y : 37.67808151245117, z : -9.96983528137207, vx : -0.02236475795507431, vy : -0.012887658551335335, vz : -0.00010744269820861518}
-LOCAL_POSITION_NED {time_boot_ms : 243729, x : 109.40962219238281, y : 37.677486419677734, z : -9.969886779785156, vx : -0.023954948410391808, vy : -0.013113063760101795, vz : -0.00017305012443102896}
-LOCAL_POSITION_NED {time_boot_ms : 243979, x : 109.40973663330078, y : 37.677276611328125, z : -9.969881057739258, vx : -0.04371815547347069, vy : -0.004350676201283932, vz : 0.0006270252051763237}
-LOCAL_POSITION_NED {time_boot_ms : 244229, x : 109.38226318359375, y : 37.69022750854492, z : -9.969354629516602, vx : -0.2798684239387512, vy : 0.10651721060276031, vz : 0.004445536062121391}
-LOCAL_POSITION_NED {time_boot_ms : 244479, x : 109.25566864013672, y : 37.74885940551758, z : -9.967907905578613, vx : -0.8151731491088867, vy : 0.35761335492134094, vz : 0.006137563847005367}
-LOCAL_POSITION_NED {time_boot_ms : 244729, x : 108.98152160644531, y : 37.87641525268555, z : -9.967415809631348, vx : -1.4317526817321777, vy : 0.6452293395996094, vz : -0.0019165349658578634}
-LOCAL_POSITION_NED {time_boot_ms : 244979, x : 108.56168365478516, y : 38.0730094909668, z : -9.968149185180664, vx : -2.0058188438415527, vy : 0.9171269536018372, vz : -0.0052565005607903}
-LOCAL_POSITION_NED {time_boot_ms : 245230, x : 108.00172424316406, y : 38.3388557434082, z : -9.969520568847656, vx : -2.558675765991211, vy : 1.1873902082443237, vz : -0.004303304478526115}
-LOCAL_POSITION_NED {time_boot_ms : 245480, x : 107.30584716796875, y : 38.671424865722656, z : -9.970559120178223, vx : -3.1081371307373047, vy : 1.4497607946395874, vz : -0.0013086843537166715}
-LOCAL_POSITION_NED {time_boot_ms : 245730, x : 106.47059631347656, y : 39.06772994995117, z : -9.970938682556152, vx : -3.657071590423584, vy : 1.6894506216049194, vz : 0.0002285348455188796}
-LOCAL_POSITION_NED {time_boot_ms : 245979, x : 105.5009536743164, y : 39.52318572998047, z : -9.971338272094727, vx : -4.182033538818359, vy : 1.926192045211792, vz : -0.0010198309319093823}
-LOCAL_POSITION_NED {time_boot_ms : 246229, x : 104.40209197998047, y : 40.04393768310547, z : -9.971746444702148, vx : -4.702603340148926, vy : 2.200885772705078, vz : 0.0009427944314666092}
-LOCAL_POSITION_NED {time_boot_ms : 246479, x : 103.16869354248047, y : 40.63428497314453, z : -9.971199035644531, vx : -5.25104284286499, vy : 2.47697114944458, vz : 0.00528548052534461}
-LOCAL_POSITION_NED {time_boot_ms : 246729, x : 101.79534912109375, y : 41.29187774658203, z : -9.969746589660645, vx : -5.810123443603516, vy : 2.742892265319824, vz : 0.00835045799612999}
-LOCAL_POSITION_NED {time_boot_ms : 246979, x : 100.2834243774414, y : 42.01643753051758, z : -9.968152046203613, vx : -6.356932163238525, vy : 3.00797700881958, vz : 0.007713631726801395}
-LOCAL_POSITION_NED {time_boot_ms : 247230, x : 98.63285064697266, y : 42.80854797363281, z : -9.967652320861816, vx : -6.869801044464111, vy : 3.2539689540863037, vz : 1.1250964234932326e-05}
-LOCAL_POSITION_NED {time_boot_ms : 247480, x : 96.86815643310547, y : 43.65073013305664, z : -9.969475746154785, vx : -7.309067249298096, vy : 3.4248688220977783, vz : -0.007807408459484577}
-LOCAL_POSITION_NED {time_boot_ms : 247730, x : 94.9985122680664, y : 44.52605056762695, z : -9.971796989440918, vx : -7.700419902801514, vy : 3.5252227783203125, vz : -0.005552171263843775}
-LOCAL_POSITION_NED {time_boot_ms : 247979, x : 93.03529357910156, y : 45.42596435546875, z : -9.972488403320312, vx : -8.041450500488281, vy : 3.617892265319824, vz : 0.0015491923550143838}
-LOCAL_POSITION_NED {time_boot_ms : 248229, x : 90.99347686767578, y : 46.35050582885742, z : -9.97187328338623, vx : -8.306546211242676, vy : 3.7119176387786865, vz : 0.0021588108502328396}
-LOCAL_POSITION_NED {time_boot_ms : 248479, x : 88.89547729492188, y : 47.29640197753906, z : -9.971541404724121, vx : -8.481266021728516, vy : 3.7974555492401123, vz : 0.00032809507683850825}
-LOCAL_POSITION_NED {time_boot_ms : 248729, x : 86.76085662841797, y : 48.263916015625, z : -9.971596717834473, vx : -8.600085258483887, vy : 3.8975508213043213, vz : 0.0003860758733935654}
-LOCAL_POSITION_NED {time_boot_ms : 248979, x : 84.5998306274414, y : 49.25948715209961, z : -9.972135543823242, vx : -8.690932273864746, vy : 4.027074337005615, vz : 0.0017167243640869856}
-LOCAL_POSITION_NED {time_boot_ms : 249229, x : 82.42630004882812, y : 50.28388977050781, z : -9.972501754760742, vx : -8.765235900878906, vy : 4.157995223999023, vz : 0.004778474103659391}
-LOCAL_POSITION_NED {time_boot_ms : 249480, x : 80.2116928100586, y : 51.346527099609375, z : -9.971891403198242, vx : -8.841904640197754, vy : 4.258383274078369, vz : 0.007347704842686653}
-LOCAL_POSITION_NED {time_boot_ms : 249730, x : 77.99267578125, y : 52.42091369628906, z : -9.970446586608887, vx : -8.91469955444336, vy : 4.3127851486206055, vz : 0.006476235575973988}
-LOCAL_POSITION_NED {time_boot_ms : 249979, x : 75.75646209716797, y : 53.50323486328125, z : -9.969244956970215, vx : -8.981115341186523, vy : 4.32005500793457, vz : 0.004069608170539141}
-LOCAL_POSITION_NED {time_boot_ms : 250229, x : 73.50287628173828, y : 54.58274459838867, z : -9.968433380126953, vx : -9.047001838684082, vy : 4.29080057144165, vz : 0.0032091387547552586}

llm/gptPlotCreator.py

@@ -0,0 +1,103 @@
+import re
+import random
+import linecache
+import subprocess
+from langchain.prompts import PromptTemplate
+from langchain.chat_models import ChatOpenAI
+from langchain.chains import LLMChain
+from langchain.llms import OpenAI
+from langchain.chains import ConversationChain
+from langchain.memory import ConversationBufferMemory
+from langchain.prompts.chat import (
+    ChatPromptTemplate,
+    HumanMessagePromptTemplate,
+)
+import os
+from dotenv import load_dotenv
+from PIL import Image
+
+
+
+class PlotCreator:
+    def __init__(self):
+        load_dotenv()
+        llm = ChatOpenAI(model_name="gpt-3.5-turbo", max_tokens=2000, temperature=0)
+        
+        mavlink_data_prompt = PromptTemplate(
+            input_variables=["human_input", "file"],
+            template="You are an AI conversation agent that will be used for generating python scripts to plot mavlink data provided by the user. Please create a python script using matplotlib and pymavlink's mavutil to plot the data provided by the user. Please do not explain the code just return the script. Please plot each independent variable over time in seconds. Please save the plot to file named plot.png with at least 400 dpi. \n\nHUMAN: {human_input} \n\nplease read this data from the file {file}.",
+        )
+        self.chain = LLMChain(verbose=True, llm=llm, prompt=mavlink_data_prompt)
+
+    @staticmethod
+    def sample_lines(filename, num_lines=5):
+        with open(filename) as f:
+            total_lines = sum(1 for _ in f)
+        
+        if total_lines < num_lines:
+            raise ValueError("File has fewer lines than the number of lines requested.")
+        
+        line_numbers = random.sample(range(1, total_lines + 1), num_lines)
+        lines = [linecache.getline(filename, line_number).rstrip() for line_number in line_numbers]
+        
+        return '\n'.join(lines)
+
+    @staticmethod
+    def extract_code_snippets(text):
+        pattern = r'```(.*?)```'
+        snippets = re.findall(pattern, text, re.DOTALL)
+        if len(snippets) == 0:
+            snippets = [text]
+        return snippets
+
+    @staticmethod
+    def write_plot_script(filename, text):
+        with open(filename, 'w') as file:
+            file.write(text)
+
+    @staticmethod
+    def attempt_to_fix_sctript(filename, error_message):
+        llm = ChatOpenAI(model_name="gpt-3.5-turbo", max_tokens=2000, temperature=0)
+
+        fix_plot_script_template = PromptTemplate(
+            input_variables=["error", "script"],
+            template="You are an AI agent that is designed to debug scripts created to plot mavlink data using matplotlib and pymavlink's mavutil. the following script produced this error: \n\n{script}\n\nThe error is: \n\n{error}\n\nPlease fix the script so that it produces the correct plot.",
+        )
+
+        # read script from file
+        with open(filename, 'r') as file:
+            script = file.read()
+        
+        chain = LLMChain(verbose=True, llm=llm, prompt=fix_plot_script_template)
+        response = chain.run({"error": error_message, "script": script})
+        print(response)
+        code = PlotCreator.extract_code_snippets(response)
+        PlotCreator.write_plot_script("plot.py", code[0])
+
+        # run the script 
+        os.system("python plot.py")
+
+    def create_plot(self, human_input):
+        file = "data/2023-01-04 20-51-25.tlog"
+
+        # prompt the user for the what plot they would like to generate
+        # human_input = input("Please enter a description of the plot you would like to generate: ")
+
+        response = self.chain.run({"file": file, "human_input": human_input})
+        print(response)
+
+        # parse the code from the response 
+        code = self.extract_code_snippets(response)
+        self.write_plot_script("plot.py", code[0])
+
+        # run the script if it doesn't work capture output and call attempt_to_fix_script
+        try:
+            subprocess.check_output(["python", "plot.py"], stderr=subprocess.STDOUT)
+        except subprocess.CalledProcessError as e:
+            print(e.output.decode())
+            self.attempt_to_fix_sctript("plot.py", e.output.decode())
+        except Exception as e:
+            print(e)
+            self.attempt_to_fix_sctript("plot.py", str(e))
+
+        return ("plot.png", None)

llm_plot.py

@@ -1,99 +1,49 @@
-import re
-import random
-import linecache
-import subprocess
-from langchain.prompts import PromptTemplate
-from langchain.chat_models import ChatOpenAI
-from langchain.chains import LLMChain
-from langchain.llms import OpenAI
-from langchain.chains import ConversationChain
-from langchain.memory import ConversationBufferMemory
-from langchain.prompts.chat import (
-    ChatPromptTemplate,
-    HumanMessagePromptTemplate,
-)
-import os
-from dotenv import load_dotenv
+import gradio as gr
+from llm.gptPlotCreator import PlotCreator
 
-# Load environment variables from .env file
-load_dotenv()
+plot_creator = PlotCreator()
 
-def sample_lines(filename, num_lines=5):
-    with open(filename) as f:
-        total_lines = sum(1 for _ in f)
-    
-    if total_lines < num_lines:
-        raise ValueError("File has fewer lines than the number of lines requested.")
-    
-    line_numbers = random.sample(range(1, total_lines + 1), num_lines)
-    lines = [linecache.getline(filename, line_number).rstrip() for line_number in line_numbers]
-    
-    return '\n'.join(lines)
-
-
-def extract_code_snippets(text):
-    pattern = r'```(.*?)```'
-    snippets = re.findall(pattern, text, re.DOTALL)
-    if len(snippets) == 0:
-        snippets = [text]
-    return snippets
+def add_text(history, text):
+    history = history + [(text, None)]
+    return history, ""
 
-def write_plot_script(filename, text):
-    with open(filename, 'w') as file:
-        file.write(text)
+def add_file(history, file):
+    history = history + [((file.name,), None)]
+    return history
 
-def attempt_to_fix_sctript(filename, error_message):
-    llm = ChatOpenAI(model_name="gpt-3.5-turbo", max_tokens=2000, temperature=0)
-
-    fix_plot_script_template = PromptTemplate(
-        input_variables=["error", "script"],
-        template="You are an AI agent that is designed to debug scripts created to plot mavlink data using matplotlib and pymavlink's mavutil. the following script produced this error: \n\n{script}\n\nThe error is: \n\n{error}\n\nPlease fix the script so that it produces the correct plot.",
-    )
-
-    # read script from file
-    with open(filename, 'r') as file:
-        script = file.read()
+def bot(history):
+    # Get the last input from the user
+    user_input = history[-1][0]
     
-    chain = LLMChain(verbose=True, llm=llm, prompt=fix_plot_script_template)
-    response = chain.run({"error": error_message, "script": script})
-    print(response)
-    code = extract_code_snippets(response)
-    write_plot_script("plot.py", code[0])
-
-    # run the script 
-    os.system("python plot.py")
-
-
-if __name__ == "__main__":
-
-
-    file = "data/2023-01-04 20-51-25.tlog"
-    llm = ChatOpenAI(model_name="gpt-3.5-turbo", max_tokens=2000, temperature=0)
+    # Check if it is a string
+    if isinstance(user_input, str):
+        # Generate the plot
+        img = plot_creator.create_plot(user_input)
+        response = img
+    else:
+        response = "**That's cool!**"
+
+    history[-1][1] = ('plot.png', None)
+    return history
+
+with gr.Blocks() as demo:
+    chatbot = gr.Chatbot([], elem_id="chatbot").style(height=750)
     
-    mavlink_data_prompt = PromptTemplate(
-        input_variables=["human_input", "file"],
-        template="You are an AI conversation agent that will be used for generating python scripts to plot mavlink data provided by the user. Please create a python script using matplotlib and pymavlink's mavutil to plot the data provided by the user. Please do not explain the code just return the script. Please plot each independent variable over time.  \n\nHUMAN: {human_input} \n\nplease read this data from the file {file}.",
+    with gr.Row():
+        with gr.Column(scale=0.85):
+            txt = gr.Textbox(
+                show_label=False,
+                placeholder="Enter text and press enter, or upload an image",
+            ).style(container=False)
+        with gr.Column(scale=0.15, min_width=0):
+            btn = gr.UploadButton("📁", file_types=["image", "video", "audio"])
+            
+    txt.submit(add_text, [chatbot, txt], [chatbot, txt]).then(
+        bot, chatbot, chatbot
+    )
+    btn.upload(add_file, [chatbot, btn], [chatbot]).then(
+        bot, chatbot, chatbot
     )
-    
-    chain = LLMChain(verbose=True, llm=llm, prompt=mavlink_data_prompt)
-
-    # prompt the user for the what plot they would like to generate
-    human_input = input("Please enter a description of the plot you would like to generate: ")
-
-    # human_input = "Please create a script to plot x y and z from LOCAL_POSITION_NED from the following data."
-    response = chain.run({"file": file, "human_input": human_input})
-    print(response)
-
-    # parse the code from the response 
-    code = extract_code_snippets(response)
-    write_plot_script("plot.py", code[0])
 
-    # run the script if it doesn't work capture output and call attempt_to_fix_script
-    try:
-        subprocess.check_output(["python", "plot.py"], stderr=subprocess.STDOUT)
-    except subprocess.CalledProcessError as e:
-        print(e.output.decode())
-        attempt_to_fix_sctript("plot.py", e.output.decode())
-    except Exception as e:
-        print(e)
-        attempt_to_fix_sctript("plot.py", str(e))
+if __name__ == "__main__":
+    demo.launch()

plot.py

@@ -1,42 +0,0 @@
-
-import matplotlib.pyplot as plt
-from pymavlink import mavutil
-
-# Open the MAVLink log file
-mlog = mavutil.mavlink_connection('data/2023-01-04 20-51-25.tlog')
-
-# Initialize lists to store the data
-time_stamps = []
-latitudes = []
-longitudes = []
-altitudes = []
-
-# Loop through the log file and extract the data
-while True:
-    msg = mlog.recv_match()
-    if not msg:
-        break
-    if msg.get_type() == 'GLOBAL_POSITION_INT':
-        time_stamps.append(msg.time_boot_ms / 1000.0)
-        latitudes.append(msg.lat / 1e7)
-        longitudes.append(msg.lon / 1e7)
-        altitudes.append(msg.alt / 1000.0)
-
-# Plot the data
-plt.plot(time_stamps, latitudes)
-plt.xlabel('Time (s)')
-plt.ylabel('Latitude')
-plt.title('Aircraft Position Over Time')
-plt.show()
-
-plt.plot(time_stamps, longitudes)
-plt.xlabel('Time (s)')
-plt.ylabel('Longitude')
-plt.title('Aircraft Position Over Time')
-plt.show()
-
-plt.plot(time_stamps, altitudes)
-plt.xlabel('Time (s)')
-plt.ylabel('Altitude (km)')
-plt.title('Aircraft Position Over Time')
-plt.show()