move to gemini-1.5-flash

persistence.log

@@ -64,3 +64,17 @@
 2024-01-11 23:38:12,386 - posthog.py -   __init__() - 20 - INFO - Anonymized telemetry enabled. See                     https://docs.trychroma.com/telemetry for more information.
 2024-01-11 23:38:14,482 - llmFactory.py -    get_llm() - 36 - DEBUG - executor3
 2024-01-11 23:38:14,482 - llmFactory.py -    get_llm() - 37 - DEBUG - {'llm_config': {'max_tokens': 1024, 'temperature': 0.1}, 'llm_type': 'geminiLLM'}
+2024-01-12 10:12:12,735 - dropbox_client.py - refresh_access_token() - 390 - INFO - Refreshing access token.
+2024-01-12 10:12:13,184 - dropbox_client.py - request_json_string_with_retry() - 474 - INFO - Request to users/get_current_account
+2024-01-12 10:12:22,103 - SentenceTransformer.py -   __init__() - 66 - INFO - Load pretrained SentenceTransformer: BAAI/bge-large-en-v1.5
+2024-01-12 10:12:27,492 - posthog.py -   __init__() - 20 - INFO - Anonymized telemetry enabled. See                     https://docs.trychroma.com/telemetry for more information.
+2024-01-12 10:12:29,074 - llmFactory.py -    get_llm() - 36 - DEBUG - executor3
+2024-01-12 10:12:29,074 - llmFactory.py -    get_llm() - 37 - DEBUG - {'llm_config': {'max_tokens': 1024, 'temperature': 0.1}, 'llm_type': 'geminiLLM'}
+2024-01-12 10:12:29,132 - proactor_events.py -   __init__() - 629 - DEBUG - Using proactor: IocpProactor
+2024-01-12 10:12:29,240 - connectionpool.py - _make_request() - 456 - DEBUG - https://app.posthog.com:443 "POST /batch/ HTTP/1.1" 200 None
+2024-01-12 10:12:29,293 - SentenceTransformer.py -   __init__() - 66 - INFO - Load pretrained SentenceTransformer: BAAI/bge-large-en-v1.5
+2024-01-12 10:12:41,220 - connectionpool.py -  _new_conn() - 1003 - DEBUG - Starting new HTTPS connection (1): device-1a455.firebaseio.com:443
+2024-01-12 10:12:41,324 - connectionpool.py - _make_request() - 456 - DEBUG - https://app.posthog.com:443 "POST /batch/ HTTP/1.1" 200 None
+2024-01-12 10:12:41,762 - connectionpool.py - _make_request() - 456 - DEBUG - https://device-1a455.firebaseio.com:443 "GET /users/131251/llm_config/executor3.json?auth=eyJ0eXAiOiAiSldUIiwgImFsZyI6ICJIUzI1NiJ9.eyJhZG1pbiI6IGZhbHNlLCAiZGVidWciOiBmYWxzZSwgInYiOiAwLCAiaWF0IjogMTcwNTAxNDc2MSwgImQiOiB7ImlkIjogIjEzMTI1MSIsICJkZWJ1ZyI6IGZhbHNlLCAiYWRtaW4iOiBmYWxzZSwgImVtYWlsIjogImFudWJoYXY3N0BnbWFpbC5jb20iLCAicHJvdmlkZXIiOiAicGFzc3dvcmQifX0.vRs8wPErJN9HLbVChqjLnOO-W7pkPq3LIVUmN1jVPGU HTTP/1.1" 200 75
+2024-01-12 10:12:41,764 - llmFactory.py -    get_llm() - 36 - DEBUG - executor3
+2024-01-12 10:12:41,764 - llmFactory.py -    get_llm() - 37 - DEBUG - {'llm_config': {'max_tokens': 1024, 'temperature': 0.1}, 'llm_type': 'geminiLLM'}

src/chromaIntf.py

@@ -1,12 +1,15 @@
 import sys
+
 try:
     import pysqlite3
+
     sys.modules["sqlite3"] = sys.modules.pop("pysqlite3")
 except:
     pass
 import chromadb
 from langchain.vectorstores import Chroma
-from chromadb.api.fastapi import requests
+
+# from chromadb.api.fastapi import requests
 from langchain.schema import Document
 from langchain.chains import RetrievalQA
 from langchain.embeddings import HuggingFaceBgeEmbeddings
@@ -21,31 +24,45 @@ from uuid import UUID
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 import logging, asyncio
 
-logger=logging.getLogger("root")
+logger = logging.getLogger("root")
+
 
 class myChromaTranslator(ChromaTranslator):
     allowed_operators = ["$and", "$or"]
     """Subset of allowed logical operators."""
-    allowed_comparators = [ "$eq","$ne","$gt","$gte","$lt","$lte",
-                           "$contains","$not_contains","$in","$nin"]
+    allowed_comparators = [
+        "$eq",
+        "$ne",
+        "$gt",
+        "$gte",
+        "$lt",
+        "$lte",
+        "$contains",
+        "$not_contains",
+        "$in",
+        "$nin",
+    ]
+
 
-class ChromaIntf():
+class ChromaIntf:
     def __init__(self):
-        self.db_interface=DbInterface()
+        self.db_interface = DbInterface()
 
         model_name = "BAAI/bge-large-en-v1.5"
-        encode_kwargs = {'normalize_embeddings': True} # set True to compute cosine similarity
+        encode_kwargs = {
+            "normalize_embeddings": True
+        }  # set True to compute cosine similarity
 
         self.embedding = HuggingFaceBgeEmbeddings(
-                model_name=model_name,
-                model_kwargs={'device': 'cpu'},  
-                encode_kwargs=encode_kwargs
+            model_name=model_name,
+            model_kwargs={"device": "cpu"},
+            encode_kwargs=encode_kwargs,
         )
 
-        self.persist_db_directory = 'db'
+        self.persist_db_directory = "db"
         self.persist_docs_directory = "persistence-docs"
         self.logger_file = "persistence.log"
-        loop=asyncio.get_event_loop()
+        loop = asyncio.get_event_loop()
         try:
             loop.run_until_complete(dbh.restoreFolder(self.persist_db_directory))
             loop.run_until_complete(dbh.restoreFolder(self.persist_docs_directory))
@@ -54,25 +71,31 @@ class ChromaIntf():
         docs = [
             Document(
                 page_content="this is test doc",
-                metadata={"timestamp":1696743148.474055,"ID":"2000-01-01 15:57:11::664165-test","source":"test"},
-                id="2000-01-01 15:57:11::664165-test"
-                ),
-            ]
+                metadata={
+                    "timestamp": 1696743148.474055,
+                    "ID": "2000-01-01 15:57:11::664165-test",
+                    "source": "test",
+                },
+                id="2000-01-01 15:57:11::664165-test",
+            ),
+        ]
 
-        self.vectorstore = Chroma.from_documents(documents=docs,
-                                  embedding=self.embedding,
-                                  persist_directory=self.persist_db_directory)
-        #self.vectorstore._client.
+        self.vectorstore = Chroma.from_documents(
+            documents=docs,
+            embedding=self.embedding,
+            persist_directory=self.persist_db_directory,
+        )
+        # self.vectorstore._client.
 
         # timestamp --> time when added
         # source --> notes/references/web/youtube/book/conversation, default conversation
         # title --> of document , will be conversation when source is conversation, default blank
         # author --> will default to blank
         #     "Year": 2024,
-        #"Month": 1,
-        #"Day": 3,
-        #"Hour": 11,
-        #"Minute": 29
+        # "Month": 1,
+        # "Day": 3,
+        # "Hour": 11,
+        # "Minute": 29
         self.metadata_field_info = [
             AttributeInfo(
                 name="timestamp",
@@ -118,12 +141,14 @@ class ChromaIntf():
                 name="author",
                 description="Author of the entry",
                 type="string",
-            )
-            ]
-        self.document_content_description = "Information to store for retrival from LLM based chatbot"
-        lf=LLMFactory()
-        #self.llm=lf.get_llm("executor2")
-        self.llm=lf.get_llm("executor3")
+            ),
+        ]
+        self.document_content_description = (
+            "Information to store for retrival from LLM based chatbot"
+        )
+        lf = LLMFactory()
+        # self.llm=lf.get_llm("executor2")
+        self.llm = lf.get_llm("executor3")
 
         self.retriever = SelfQueryRetriever.from_llm(
             self.llm,
@@ -131,63 +156,61 @@ class ChromaIntf():
             self.document_content_description,
             self.metadata_field_info,
             structured_query_translator=ChromaTranslator(),
-            verbose=True
+            verbose=True,
         )
 
-
-    async def getRelevantDocs(self,query:str,kwargs:dict):
+    async def getRelevantDocs(self, query: str, kwargs: dict):
         """This should also post the result to firebase"""
-        print("retriver state",self.retriever.search_kwargs)
-        print("retriver state",self.retriever.search_type)
+        print("retriver state", self.retriever.search_kwargs)
+        print("retriver state", self.retriever.search_type)
         try:
             for key in kwargs.keys():
                 if "search_type" in key:
-                    self.retriever.search_type=kwargs[key]
+                    self.retriever.search_type = kwargs[key]
                 else:
-                    self.retriever.search_kwargs[key]=kwargs[key]
+                    self.retriever.search_kwargs[key] = kwargs[key]
         except:
             print("setting search args failed")
         print("reaching step2")
         try:
-            #loop=asyncio.get_event_loop()
-            retVal=self.retriever.get_relevant_documents(query)
+            # loop=asyncio.get_event_loop()
+            retVal = self.retriever.get_relevant_documents(query)
         except Exception as ex:
-            logger.exception("Exception occured:",exc_info=True)
-        value=[]
-        excludeMeta=True
+            logger.exception("Exception occured:", exc_info=True)
+        value = []
+        excludeMeta = True
         print("reaching step3")
         print(str(len(retVal)))
         print("reaching step4")
         try:
             for item in retVal:
                 if excludeMeta:
-                    v=item.page_content+" \n"
+                    v = item.page_content + " \n"
                 else:
-                    v="Info:"+item.page_content+" "
+                    v = "Info:" + item.page_content + " "
                     for key in item.metadata.keys():
                         if key != "ID":
-                            v+=key+":"+str(item.metadata[key])+" "
+                            v += key + ":" + str(item.metadata[key]) + " "
                 value.append(v)
             print("reaching step5")
-            self.db_interface.add_to_cache(input=query,value=value)
+            self.db_interface.add_to_cache(input=query, value=value)
         except:
             print("reaching step6")
             for item in retVal:
                 if excludeMeta:
-                    v=item['page_content']+" \n"
+                    v = item["page_content"] + " \n"
                 else:
-                    v="Info:"+item['page_content']+" "
-                    for key in item['metadata'].keys():
+                    v = "Info:" + item["page_content"] + " "
+                    for key in item["metadata"].keys():
                         if key != "ID":
-                            v+=key+":"+str(item['metadata'][key])+" "
+                            v += key + ":" + str(item["metadata"][key]) + " "
                 value.append(v)
             print("reaching step7")
-            self.db_interface.add_to_cache(input=query,value=value)
+            self.db_interface.add_to_cache(input=query, value=value)
         print("reaching step8")
         return retVal
-    
 
-    async def addText(self,inStr:str,metadata):
+    async def addText(self, inStr: str, metadata):
         # metadata expected is some of following
         # timestamp --> time when added
         # source --> notes/references/web/youtube/book/conversation, default conversation
@@ -195,78 +218,83 @@ class ChromaIntf():
         # author --> will default to blank
 
         ##TODO: Preprocess inStr to remove any html, markdown tags etc.
-        metadata=metadata.dict()
+        metadata = metadata.dict()
         if "timestamp" not in metadata.keys():
-            metadata['timestamp']=datetime.now().isoformat()
+            metadata["timestamp"] = datetime.now().isoformat()
         else:
-            metadata['timestamp']=datetime.fromisoformat(metadata['timestamp'])
+            metadata["timestamp"] = datetime.fromisoformat(metadata["timestamp"])
             pass
         if "source" not in metadata.keys():
-            metadata['source']="conversation"
-        if "title" not in metadata.keys(): 
+            metadata["source"] = "conversation"
+        if "title" not in metadata.keys():
             metadata["title"] = ""
-        if  metadata["source"] == "conversation":
+        if metadata["source"] == "conversation":
             metadata["title"] == "conversation"
-        if "author" not in metadata.keys(): 
+        if "author" not in metadata.keys():
             metadata["author"] = ""
-        
-        #TODO: If url is present in input or when the splitting need to be done, then we'll need to change how we 
+
+        # TODO: If url is present in input or when the splitting need to be done, then we'll need to change how we
         # formulate the ID and may be filename to store information
-        metadata['ID']=metadata['timestamp'].strftime("%Y-%m-%d %H-%M-%S")+"-"+metadata['title']
-        metadata['Year']=metadata['timestamp'].year
-        metadata['Month']=metadata['timestamp'].month
-        metadata['Day']=int(metadata['timestamp'].strftime("%d"))
-        metadata['Hour']=metadata['timestamp'].hour
-        metadata['Minute']=metadata['timestamp'].minute
-        metadata['timestamp']=metadata['timestamp'].isoformat()
+        metadata["ID"] = (
+            metadata["timestamp"].strftime("%Y-%m-%d %H-%M-%S")
+            + "-"
+            + metadata["title"]
+        )
+        metadata["Year"] = metadata["timestamp"].year
+        metadata["Month"] = metadata["timestamp"].month
+        metadata["Day"] = int(metadata["timestamp"].strftime("%d"))
+        metadata["Hour"] = metadata["timestamp"].hour
+        metadata["Minute"] = metadata["timestamp"].minute
+        metadata["timestamp"] = metadata["timestamp"].isoformat()
         print("Metadata is:")
         print(metadata)
-        #md.pop("timestamp")
-        with open("./docs/"+metadata['ID']+".txt","w") as fd:
+        # md.pop("timestamp")
+        with open("./docs/" + metadata["ID"] + ".txt", "w") as fd:
             fd.write(inStr)
             print("written to file", inStr)
         text_splitter = RecursiveCharacterTextSplitter(
             chunk_size=800,
             chunk_overlap=50,
             length_function=len,
-            is_separator_regex=False)
-        #docs = [    Document(page_content=inStr, metadata=metadata)]
-        docs=text_splitter.create_documents([inStr],[metadata])
-        partNumber=0
+            is_separator_regex=False,
+        )
+        # docs = [    Document(page_content=inStr, metadata=metadata)]
+        docs = text_splitter.create_documents([inStr], [metadata])
+        partNumber = 0
         for doc in docs:
             if partNumber > 0:
-                doc.metadata['ID']+=f"__{partNumber}"
-            partNumber+=1
+                doc.metadata["ID"] += f"__{partNumber}"
+            partNumber += 1
             print(f"{partNumber} follows:")
             print(doc)
         try:
-            print(metadata['ID'])
-            ids=[doc.metadata['ID'] for doc in docs]
+            print(metadata["ID"])
+            ids = [doc.metadata["ID"] for doc in docs]
             print("ids are:")
             print(ids)
-            return await self.vectorstore.aadd_documents(docs,ids=ids)
+            return await self.vectorstore.aadd_documents(docs, ids=ids)
         except Exception as ex:
-            logger.exception("exception in adding",exc_info=True)
+            logger.exception("exception in adding", exc_info=True)
             print("inside expect of addText")
-            return await self.vectorstore.aadd_documents(docs,ids=[metadata.ID])
-        
+            return await self.vectorstore.aadd_documents(docs, ids=[metadata.ID])
+
     async def listDocs(self):
-        collection=self.vectorstore._client.get_collection(self.vectorstore._LANGCHAIN_DEFAULT_COLLECTION_NAME,embedding_function=self.embedding)
+        collection = self.vectorstore._client.get_collection(
+            self.vectorstore._LANGCHAIN_DEFAULT_COLLECTION_NAME,
+            embedding_function=self.embedding,
+        )
         return collection.get()
-        #return self.vectorstore._client._get(collection_id=self._uuid(collectionInfo.id))
-        
-        
+        # return self.vectorstore._client._get(collection_id=self._uuid(collectionInfo.id))
+
     async def persist(self):
         self.vectorstore.persist()
         await dbh.backupFile(self.logger_file)
         await dbh.backupFolder(self.persist_db_directory)
         return await dbh.backupFolder(self.persist_docs_directory)
-    
-    def _uuid(self,uuid_str: str) -> UUID:
+
+    def _uuid(self, uuid_str: str) -> UUID:
         try:
             return UUID(uuid_str)
         except ValueError:
             print("Error generating uuid")
             raise ValueError(f"Could not parse {uuid_str} as a UUID")
-
-    

src/llm/geminiLLM.py

@@ -1,34 +1,33 @@
-from typing import Any, List, Mapping, Optional, Dict 
-from pydantic import Extra, Field #, root_validator, model_validator
-import os,json
+from typing import Any, List, Mapping, Optional, Dict
+from pydantic import Extra, Field  # , root_validator, model_validator
+import os, json
 from langchain.callbacks.manager import CallbackManagerForLLMRun
 from langchain.llms.base import LLM
 import google.generativeai as genai
 from google.generativeai import types
 import ast
-#from langchain.llms import GooglePalm
-import requests,logging
 
-logger=logging.getLogger("llm")
+# from langchain.llms import GooglePalm
+import requests, logging
+
+logger = logging.getLogger("llm")
+
 
 class GeminiLLM(LLM):
-    
-    model_name: str = "gemini-pro"
+
+    model_name: str = "gemini-1.5-flash"  # "gemini-pro"
     temperature: float = 0
     max_tokens: int = 2048
     stop: Optional[List] = []
-    prev_prompt: Optional[str]=""
-    prev_stop: Optional[str]=""
-    prev_run_manager:Optional[Any]=None
-    model: Optional[Any]=None
+    prev_prompt: Optional[str] = ""
+    prev_stop: Optional[str] = ""
+    prev_run_manager: Optional[Any] = None
+    model: Optional[Any] = None
 
-    def __init__(
-        self,
-        **kwargs
-    ):
+    def __init__(self, **kwargs):
         super().__init__(**kwargs)
-        self.model=genai.GenerativeModel(self.model_name)
-        #self.model = palm.Text2Text(self.model_name)
+        self.model = genai.GenerativeModel(self.model_name)
+        # self.model = palm.Text2Text(self.model_name)
 
     @property
     def _llm_type(self) -> str:
@@ -40,76 +39,95 @@ class GeminiLLM(LLM):
         stop: Optional[List[str]] = None,
         run_manager: Optional[CallbackManagerForLLMRun] = None,
     ) -> str:
-        self.prev_prompt=prompt
-        self.prev_stop=stop
-        self.prev_run_manager=run_manager
-        #print(types.SafetySettingDict)
+        self.prev_prompt = prompt
+        self.prev_stop = stop
+        self.prev_run_manager = run_manager
+        # print(types.SafetySettingDict)
         if stop == None:
-            stop=self.stop
-        logger.debug("\nLLM in use is:" +self._llm_type)
-        logger.debug("Request to LLM is "+prompt)
-        
-        response=self.model.generate_content(prompt,
-                                    generation_config={"stop_sequences":self.stop,
-                                    "temperature":self.temperature, "max_output_tokens":self.max_tokens},
-                            safety_settings=[{"category":"HARM_CATEGORY_SEXUALLY_EXPLICIT","threshold":"BLOCK_NONE"},
-                             {"category":"HARM_CATEGORY_HATE_SPEECH","threshold":"BLOCK_NONE"},
-                             {"category":"HARM_CATEGORY_HARASSMENT","threshold":"BLOCK_NONE"},
-                             {"category":"HARM_CATEGORY_DANGEROUS_CONTENT","threshold":"BLOCK_NONE"}],
-                             stream=False
-                        )
+            stop = self.stop
+        logger.debug("\nLLM in use is:" + self._llm_type)
+        logger.debug("Request to LLM is " + prompt)
+
+        response = self.model.generate_content(
+            prompt,
+            generation_config={
+                "stop_sequences": self.stop,
+                "temperature": self.temperature,
+                "max_output_tokens": self.max_tokens,
+            },
+            safety_settings=[
+                {
+                    "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
+                    "threshold": "BLOCK_NONE",
+                },
+                {"category": "HARM_CATEGORY_HATE_SPEECH", "threshold": "BLOCK_NONE"},
+                {"category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_NONE"},
+                {
+                    "category": "HARM_CATEGORY_DANGEROUS_CONTENT",
+                    "threshold": "BLOCK_NONE",
+                },
+            ],
+            stream=False,
+        )
         try:
-            val=response.text
+            val = response.text
             if val == None:
                 logger.debug("Response from LLM was None\n")
-                filterStr=""
+                filterStr = ""
                 for item in response.filters:
-                    for key,val in item.items():
-                        filterStr+=key+":"+str(val)
-                logger.error("Will switch to fallback LLM as response from palm is None::"+filterStr)
-                raise(Exception)
+                    for key, val in item.items():
+                        filterStr += key + ":" + str(val)
+                logger.error(
+                    "Will switch to fallback LLM as response from palm is None::"
+                    + filterStr
+                )
+                raise (Exception)
             else:
-                logger.debug("Response from LLM "+val)
+                logger.debug("Response from LLM " + val)
         except Exception as ex:
-                logger.error("Will switch to fallback LLM as response from palm is None::")
-                raise(Exception)
+            logger.error("Will switch to fallback LLM as response from palm is None::")
+            raise (Exception)
         if run_manager:
             pass
-            #run_manager.on_llm_end(val)
+            # run_manager.on_llm_end(val)
         return val
-    
+
     @property
     def _identifying_params(self) -> Mapping[str, Any]:
         """Get the identifying parameters."""
-        return {"name": self.model_name, "type": "palm"} 
-    
-    def extractJson(self,val:str) -> Any:
+        return {"name": self.model_name, "type": "palm"}
+
+    def extractJson(self, val: str) -> Any:
         """Helper function to extract json from this LLMs output"""
-        #This is assuming the json is the first item within ````
+        # This is assuming the json is the first item within ````
         # palm is responding always with ```json and ending with ```, however sometimes response is not complete
         # in case trailing ``` is not seen, we will call generation again with prev_prompt and result appended to it
         try:
-            count=0
-            while val.startswith("```json") and not val.endswith("```") and count<7:
-                val=self._call(prompt=self.prev_prompt+" "+val,stop=self.prev_stop,run_manager=self.prev_run_manager)
-                count+=1
-            v2=val.replace("```json","```").split("```")[1]
+            count = 0
+            while val.startswith("```json") and not val.endswith("```") and count < 7:
+                val = self._call(
+                    prompt=self.prev_prompt + " " + val,
+                    stop=self.prev_stop,
+                    run_manager=self.prev_run_manager,
+                )
+                count += 1
+            v2 = val.replace("```json", "```").split("```")[1]
             try:
-                v4=json.loads(v2)
+                v4 = json.loads(v2)
             except:
-                #v3=v2.replace("\n","").replace("\r","").replace("'","\"")
-                v3=json.dumps(ast.literal_eval(v2))
-                v4=json.loads(v3)
+                # v3=v2.replace("\n","").replace("\r","").replace("'","\"")
+                v3 = json.dumps(ast.literal_eval(v2))
+                v4 = json.loads(v3)
         except:
-            v2=val.replace("\n","").replace("\r","")
-            v3=json.dumps(ast.literal_eval(val))
-            #v3=v2.replace("'","\"")
-            v4=json.loads(v3)
-            #v4=json.loads(v2)
+            v2 = val.replace("\n", "").replace("\r", "")
+            v3 = json.dumps(ast.literal_eval(val))
+            # v3=v2.replace("'","\"")
+            v4 = json.loads(v3)
+            # v4=json.loads(v2)
         return v4
-    
-    def extractPython(self,val:str) -> Any:
+
+    def extractPython(self, val: str) -> Any:
         """Helper function to extract python from this LLMs output"""
-        #This is assuming the python is the first item within ````
-        v2=val.replace("```python","```").split("```")[1]
-        return v2
+        # This is assuming the python is the first item within ````
+        v2 = val.replace("```python", "```").split("```")[1]
+        return v2