First batch of files

.gitignore

@@ -0,0 +1,160 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

README.md

@@ -1,7 +1,7 @@
 ---
 title: Rag Demo 1
-emoji: 🌍
-colorFrom: green
+emoji: 🏆
+colorFrom: purple
 colorTo: indigo
 sdk: gradio
 sdk_version: 3.47.1
@@ -10,3 +10,15 @@ pinned: false
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+# Books sourced from public domain. 
+1. Literature
+    1. https://www.planetebook.com/
+1. Finance 
+    1. https://assets.openstax.org/oscms-prodcms/media/documents/PrinciplesofFinance-WEB.pdf
+    1. http://www.untag-smd.ac.id/files/Perpustakaan_Digital_1/FINANCE%20Essencial%20finance.pdf
+    1. https://www.drnishikantjha.com/booksCollection/CoreCourseFinancialAccounting%20.pdf
+	1. https://link.springer.com/book/10.1007/978-3-030-74817-3
+	1. https://openbooks.lib.msu.edu/financialmanagement/
+	1. https://press.rebus.community/financialstrategy/
+1. https://open.umn.edu/opentextbooks/subjects/finance

app.ipynb

@@ -0,0 +1,213 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Setup UI"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Loading settings from ../../env/ai.json\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import json\n",
+    "\n",
+    "# If the file does not exist it'll default to the manual setting see below\n",
+    "filePathToSettingsFile = '../../env/ai.json'\n",
+    "\n",
+    "# Is there a settings file? \n",
+    "if os.path.exists(filePathToSettingsFile):\n",
+    "    # Yes there is so load settings from there\n",
+    "    \n",
+    "    print(f'Loading settings from {filePathToSettingsFile}')\n",
+    "    f = open(filePathToSettingsFile)\n",
+    "    settingsJson = json.load(f)\n",
+    "    del f\n",
+    "\n",
+    "    for key in settingsJson:\n",
+    "        os.environ[key] = settingsJson[key]\n",
+    "        \n",
+    "    del settingsJson\n",
+    "else:        \n",
+    "    # Set variables manually\n",
+    "    \n",
+    "    print('Setting variables manually as there is not ai.json settings file')\n",
+    "\n",
+    "    # Update the variables below with your own settings\n",
+    "    os.environ['REQUESTS_CA_BUNDLE'] = '../../env/ZCert.pem'    \n",
+    "    os.environ['HUGGING_FACE_API_KEY'] = 'Get here: https://huggingface.co/settings/tokens'\n",
+    "    os.environ['OPENAI_API_KEY'] = 'Get here: https://platform.openai.com/account/api-keys'\n",
+    "    os.environ[\"SERPAPI_API_KEY\"] = 'serpapi KEY, Get here: https://serpapi.com/manage-api-key'    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "c:\\sc\\ai\\rag-demo-1\\.venv\\lib\\site-packages\\pinecone\\index.py:4: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from tqdm.autonotebook import tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import pinecone\n",
+    "from langchain.chains import RetrievalQA\n",
+    "from langchain.embeddings import OpenAIEmbeddings\n",
+    "from langchain.llms import OpenAI\n",
+    "from langchain.vectorstores import Pinecone\n",
+    "import gradio as gr"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "c:\\sc\\ai\\rag-demo-1\\.venv\\lib\\site-packages\\langchain\\vectorstores\\pinecone.py:59: UserWarning: Passing in `embedding` as a Callable is deprecated. Please pass in an Embeddings object instead.\n",
+      "  warnings.warn(\n"
+     ]
+    }
+   ],
+   "source": [
+    "embeddings = OpenAIEmbeddings()\n",
+    "\n",
+    "course = \"Roman History\"\n",
+    "if course == \"Roman History\":\n",
+    "    pinecone.init(api_key=os.environ[\"PINECONE_API_KEY_2\"], environment=os.environ[\"PINECONE_API_ENV_2\"])\n",
+    "    index = pinecone.Index(\"rag-demo-1-history-rome\")\n",
+    "    vector_store = Pinecone(index, embeddings.embed_query, \"text\")\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "llm = OpenAI(temperature=0)\n",
+    "qa = RetrievalQA.from_chain_type(llm=llm, chain_type=\"stuff\", retriever=vector_store.as_retriever())\n",
+    "\n",
+    "query = input(\"Who was ceasar and why whas he noteable?\")\n",
+    "answer = qa.run(query)\n",
+    "# print(answer)\n",
+    "# print(\"\\nWhat else can I help you with:\")\n",
+    "\n",
+    "# while True:\n",
+    "#     query = input(\"Who was ceasar and why whas he notable?\")\n",
+    "#     answer = qa.run(query)\n",
+    "#     print(answer)\n",
+    "#     print(\"\\nWhat else can I help you with:\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "qa = RetrievalQA.from_chain_type(llm=OpenAI(temperature=0), chain_type=\"stuff\", retriever=vector_store.as_retriever())\n",
+    "\n",
+    "print(\"Connector development help bot. What do you want to know?\")\n",
+    "while True:\n",
+    "    query = input(\"\")\n",
+    "    answer = qa.run(query)\n",
+    "    print(answer)\n",
+    "    print(\"\\nWhat else can I help you with:\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Running on local URL:  http://127.0.0.1:7860\n",
+      "\n",
+      "To create a public link, set `share=True` in `launch()`.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div><iframe src=\"http://127.0.0.1:7860/\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": []
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "def ask_question(openAIKey, prompt, question):\n",
+    "    return \"Your question is \" + question + \"!!\"\n",
+    "\n",
+    "iface = gr.Interface(fn=ask_question, \n",
+    "                     inputs=[\n",
+    "                            gr.Textbox(label=\"OpenAI Key\", placeholder=\"Paste your OpenAI key here\"),\n",
+    "                            gr.TextArea(label=\"Prompt\"), \n",
+    "                            gr.TextArea(label=\"Question\", placeholder=\"Type your question here, it will get injected into the prompt at QUESTION\"),\n",
+    "                            ],\n",
+    "                     outputs=\"text\")\n",
+    "\n",
+    "iface.launch()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

app.py

@@ -0,0 +1,44 @@
+import os
+import gradio as gr
+from huggingface_hub import Repository
+
+def greet(name):
+    # returnValue = "Hello " + name + "!!"
+    returnValue = ""
+    
+    textFilename = f'./data/test.txt'
+    
+    try:
+                
+        returnValue = os.listdir(name)
+            
+    except Exception as err:
+        returnValue = str(err)
+        
+    return returnValue
+    
+    # try:
+    
+    #     with open(textFilename, 'w') as f:
+    #         f.write(returnValue)
+            
+    # except Exception as err:
+    #     returnValue = str(err)
+    ##################################
+    # try:
+    
+    #     file = open(textFilename, "r")
+    #     content = file.read()        
+    #     returnValue = content
+    #     file.close()        
+            
+    # except Exception as err:
+    #     returnValue = str(err)
+        
+    # return returnValue
+
+iface = gr.Interface(fn=greet, 
+                     inputs=gr.TextArea(label="Name"), 
+                     outputs="text")
+
+iface.launch()

housekeeping.ipynb

@@ -0,0 +1,100 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Loading settings from ../../env/ai.json\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import json\n",
+    "\n",
+    "# If the file does not exist it'll default to the manual setting see below\n",
+    "filePathToSettingsFile = '../../env/ai.json'\n",
+    "\n",
+    "# Is there a settings file? \n",
+    "if os.path.exists(filePathToSettingsFile):\n",
+    "    # Yes there is so load settings from there\n",
+    "    \n",
+    "    print(f'Loading settings from {filePathToSettingsFile}')\n",
+    "    f = open(filePathToSettingsFile)\n",
+    "    settingsJson = json.load(f)\n",
+    "    del f\n",
+    "\n",
+    "    for key in settingsJson:\n",
+    "        os.environ[key] = settingsJson[key]\n",
+    "        \n",
+    "    del settingsJson\n",
+    "else:        \n",
+    "    # Set variables manually\n",
+    "    \n",
+    "    print('Setting variables manually as there is not ai.json settings file')\n",
+    "\n",
+    "    # Update the variables below with your own settings\n",
+    "    os.environ['REQUESTS_CA_BUNDLE'] = '../../env/ZCert.pem'    \n",
+    "    os.environ['HUGGING_FACE_API_KEY'] = 'Get here: https://huggingface.co/settings/tokens'\n",
+    "    os.environ['OPENAI_API_KEY'] = 'Get here: https://platform.openai.com/account/api-keys'\n",
+    "    os.environ[\"SERPAPI_API_KEY\"] = 'serpapi KEY, Get here: https://serpapi.com/manage-api-key'\n",
+    "    os.environ[\"PINECONE_API_KEY\"] = 'serpapi KEY, Get here: https://app.pinecone.io/'\n",
+    "    os.environ[\"PINECONE_API_ENV\"] = 'serpapi KEY, Get here: https://app.pinecone.io/'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from huggingface_hub import HfFileSystem\n",
+    "fs = HfFileSystem()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0\n"
+     ]
+    }
+   ],
+   "source": [
+    "list = fs.glob(\"thrag/rag-demo-1/*\")\n",
+    "print(len(list))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

vector-stores-create-1.ipynb

@@ -0,0 +1,262 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Loading settings from ../../env/ai.json\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import json\n",
+    "\n",
+    "# If the file does not exist it'll default to the manual setting see below\n",
+    "filePathToSettingsFile = '../../env/ai.json'\n",
+    "\n",
+    "# Is there a settings file? \n",
+    "if os.path.exists(filePathToSettingsFile):\n",
+    "    # Yes there is so load settings from there\n",
+    "    \n",
+    "    print(f'Loading settings from {filePathToSettingsFile}')\n",
+    "    f = open(filePathToSettingsFile)\n",
+    "    settingsJson = json.load(f)\n",
+    "    del f\n",
+    "\n",
+    "    for key in settingsJson:\n",
+    "        os.environ[key] = settingsJson[key]\n",
+    "        \n",
+    "    del settingsJson\n",
+    "else:        \n",
+    "    # Set variables manually\n",
+    "    \n",
+    "    print('Setting variables manually as there is not ai.json settings file')\n",
+    "\n",
+    "    # Update the variables below with your own settings\n",
+    "    os.environ['REQUESTS_CA_BUNDLE'] = '../../env/ZCert.pem'    \n",
+    "    os.environ['HUGGING_FACE_API_KEY'] = 'Get here: https://huggingface.co/settings/tokens'\n",
+    "    os.environ['OPENAI_API_KEY'] = 'Get here: https://platform.openai.com/account/api-keys'\n",
+    "    os.environ[\"SERPAPI_API_KEY\"] = 'serpapi KEY, Get here: https://serpapi.com/manage-api-key'    "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Load data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# https://github.com/gkamradt/langchain-tutorials/blob/main/data_generation/Ask%20A%20Book%20Questions.ipynb\n",
+    "from langchain.document_loaders import UnstructuredPDFLoader, OnlinePDFLoader, PyPDFLoader\n",
+    "from langchain.text_splitter import RecursiveCharacterTextSplitter"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 57,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import glob\n",
+    "from pdfminer.high_level import extract_text\n",
+    "\n",
+    "rootFolder = '../rag-demo-1-data/'\n",
+    "literatureFolder = 'literature/'\n",
+    "historyOfRomeFolder = 'history-roman/'\n",
+    "\n",
+    "currentFolder = f'{rootFolder}{literatureFolder}'\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 58,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for filename in glob.glob(f\"{currentFolder}*.pdf\"):\n",
+    "\n",
+    "    print(f'About to extract {filename}')\n",
+    "    try:\n",
+    "        text = extract_text(filename)\n",
+    "        text = text.encode('ascii', errors='ignore').decode('ascii')\n",
+    "\n",
+    "        textFilename = f'{filename}.txt'\n",
+    "        print(textFilename)\n",
+    "        with open(textFilename, 'w') as f:\n",
+    "            f.write(text)\n",
+    "            \n",
+    "        os.rename(filename, f\"{filename}.done\")\n",
+    "    except Exception as err:\n",
+    "        print(f\"Error with file {filename} {err}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 59,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.document_loaders import DirectoryLoader\n",
+    "from langchain.document_loaders import TextLoader\n",
+    "\n",
+    "loader = DirectoryLoader(currentFolder, glob=\"**/*.txt\", loader_cls=TextLoader)\n",
+    "docs = loader.load()\n",
+    "\n",
+    "print(len(docs))\n",
+    "print(len(docs[0].page_content))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 61,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "16596\n"
+     ]
+    }
+   ],
+   "source": [
+    "text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)\n",
+    "texts = text_splitter.split_documents(docs)\n",
+    "print(len(texts))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 62,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'source': '..\\\\rag-demo-1-data\\\\literature\\\\moby-dick.pdf.txt'}\n"
+     ]
+    }
+   ],
+   "source": [
+    "#print(texts[8000].page_content)\n",
+    "print(texts[8000].metadata)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Create embeddings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 63,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.vectorstores import Pinecone\n",
+    "from langchain.embeddings import OpenAIEmbeddings\n",
+    "import pinecone\n",
+    "\n",
+    "embeddings = OpenAIEmbeddings(openai_api_key=os.environ['OPENAI_API_KEY'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 64,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "index_name = \"\" # put in the name of your pinecone index here\n",
+    "pinecone.init(api_key='', environment='gcp-starter')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 65,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "docsearch = Pinecone.from_documents(texts, embeddings, index_name=index_name)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 66,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "query = \"What is moby dick?\"\n",
+    "searchResult = docsearch.similarity_search(query, k=5)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 67,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "..\\rag-demo-1-data\\literature\\moby-dick.pdf.txt\n",
+      "Moby Dick By Herman MelvilleDownload free eBooks of classic literature, books and \n",
+      "novels at Planet eBook. Subscribe to our free eBooks blog \n",
+      "and email newsletter.\fETYMOLOGY.(Supplied  by  a  Late  Consumptive  Usher  to  a  Grammar \n",
+      "School)The  pale  Usherthreadbare  in  coat,  heart,  body,  and \n",
+      "brain; I see him now. He was ever dusting his old lexicons \n",
+      "and grammars, with a queer handkerchief, mockingly em-\n",
+      "bellished with all the gay flags of all the known nations of \n",
+      "the world. He loved to dust his old grammars; it somehow \n",
+      "mildly reminded him of his mortality.While you take in hand to school others, and to teach them \n",
+      "by what name a whale-fish is to be called in our tongue \n",
+      "leaving out, through ignorance, the letter H, which almost \n",
+      "alone maketh the signification of the word, you deliver that \n",
+      "which is not true. HACKLUYTWHALE.  Sw. and Dan. HVAL. This animal is named from \n",
+      "roundness or rolling; for in Dan. HVALT is arched or vaulted.\n"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "print(searchResult[0].metadata['source'])\n",
+    "print(searchResult[0].page_content)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

vector-stores-query.ipynb

@@ -0,0 +1,140 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Loading settings from ../../env/ai.json\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import json\n",
+    "\n",
+    "# If the file does not exist it'll default to the manual setting see below\n",
+    "filePathToSettingsFile = '../../env/ai.json'\n",
+    "\n",
+    "# Is there a settings file? \n",
+    "if os.path.exists(filePathToSettingsFile):\n",
+    "    # Yes there is so load settings from there\n",
+    "    \n",
+    "    print(f'Loading settings from {filePathToSettingsFile}')\n",
+    "    f = open(filePathToSettingsFile)\n",
+    "    settingsJson = json.load(f)\n",
+    "    del f\n",
+    "\n",
+    "    for key in settingsJson:\n",
+    "        os.environ[key] = settingsJson[key]\n",
+    "        \n",
+    "    del settingsJson\n",
+    "else:        \n",
+    "    # Set variables manually\n",
+    "    \n",
+    "    print('Setting variables manually as there is not ai.json settings file')\n",
+    "\n",
+    "    # Update the variables below with your own settings\n",
+    "    os.environ['REQUESTS_CA_BUNDLE'] = '../../env/ZCert.pem'    \n",
+    "    os.environ['HUGGING_FACE_API_KEY'] = 'Get here: https://huggingface.co/settings/tokens'\n",
+    "    os.environ['OPENAI_API_KEY'] = 'Get here: https://platform.openai.com/account/api-keys'\n",
+    "    os.environ[\"SERPAPI_API_KEY\"] = 'serpapi KEY, Get here: https://serpapi.com/manage-api-key'    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.vectorstores import Pinecone\n",
+    "from langchain.embeddings import OpenAIEmbeddings\n",
+    "import pinecone\n",
+    "\n",
+    "embeddings = OpenAIEmbeddings(openai_api_key=os.environ['OPENAI_API_KEY'])\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "index_name = \"rag-demo-1-history-rome\"\n",
+    "text_field = \"text\"\n",
+    "\n",
+    "index = pinecone.Index(index_name)\n",
+    "vectorstore = Pinecone(\n",
+    "    index, embeddings.embed_query, text_field\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "query = \"When was Ceasar born?\"\n",
+    "\n",
+    "result = vectorstore.similarity_search(\n",
+    "    query,  # our search query\n",
+    "    k=3  # return 3 most relevant docs\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "3\n",
+      "page_content='The situation in Rome. While Pompey was adding to his\\nmilitary reputation in the East he was regarded with jealous\\nand anxious eyes not only by the Senate but also by the other\\nchampions of the popular party, Crassus who found his wealth\\nno match for Pompeys military achievements, and Caius Julius\\nCaesar who was rapidly coming to be one of the leading figures in\\nRoman public life. Caesar was born in 100 B. C., of the patrician [162]\\ngens of the Julii, but since his aunt was the wife of Marius,\\nand he himself had married the daughter of Cinna, his lot was\\ncast with the Populares. As a young man he had distinguished\\nhimself by refusing to divorce his wife at Sullas behest, whereat\\nSulla was with difficulty induced to spare his life, saying that he\\nsaw in him many a Marius. For the time being Caesar judged it\\nprudent to withdraw from Rome to Rhodes. While in the East\\nhe was captured by pirates, and after being ransomed, fulfilled' metadata={'source': '..\\\\rag-demo-1-data\\\\history-roman\\\\3. A History of Rome to 565 A. D. author Arthur Edward Romilly Boak.pdf.txt'}\n",
+      "##\n",
+      "page_content='13 After the adoption his full name was Caius Julius Caesar Octavianus.\\nAlthough he was known as Caesar by his contemporaries, it is more convenient\\nto refer to him henceforth as Octavian, to distinguish him from his adoptive\\nfather.\\n\\n[187]\\n\\n\\x0c222\\n\\nA History of Rome to 565 A. D.\\n\\nCaesars murder, was about to leave Italy to join Brutus when he\\nheard of the changed situation in Rome and returned to assume\\nthe leadership of the republican party. Antony left Rome for\\nthe Cisalpine province early in December, 44 B. C., and Cicero\\ninduced the Senate to enter into a coalition with Octavian against\\nhim.\\nIn his Philippic Orations he gave full vent to his bitter\\nhatred of Antony and so aroused the latters undying enmity.' metadata={'source': '..\\\\rag-demo-1-data\\\\history-roman\\\\3. A History of Rome to 565 A. D. author Arthur Edward Romilly Boak.pdf.txt'}\n",
+      "##\n",
+      "page_content='[180]\\n\\nIV. THE DICTATORSHIP OF JULIUS CAESAR: 4644 B. C.\\n\\n213\\n\\n46, when he celebrated his triumph over the Gauls and his other\\nnon-Roman enemies. He assumed it again after Munda in the\\nfollowing year.' metadata={'source': '..\\\\rag-demo-1-data\\\\history-roman\\\\3. A History of Rome to 565 A. D. author Arthur Edward Romilly Boak.pdf.txt'}\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(len(result))\n",
+    "\n",
+    "print(result[0])\n",
+    "print('##')\n",
+    "print(result[1])\n",
+    "print('##')\n",
+    "print(result[2])"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}