initial working code

.env.example

@@ -0,0 +1,87 @@
+# LLM_MODEL_TYPE=openai
+# LLM_MODEL_TYPE=gpt4all-j
+# LLM_MODEL_TYPE=gpt4all
+# LLM_MODEL_TYPE=llamacpp
+# LLM_MODEL_TYPE=huggingface
+# LLM_MODEL_TYPE=mosaicml
+# LLM_MODEL_TYPE=stablelm
+# LLM_MODEL_TYPE=openllm
+LLM_MODEL_TYPE=hftgi
+
+OPENLLM_SERVER_URL=
+
+HFTGI_SERVER_URL=https://enabled-factually-cougar.ngrok-free.app
+
+OPENAI_API_KEY=
+
+# if unset, default to "gpt-3.5-turbo"
+OPENAI_MODEL_NAME=
+
+# cpu, mps or cuda:0 - if unset, use whatever detected
+HF_EMBEDDINGS_DEVICE_TYPE=
+HF_PIPELINE_DEVICE_TYPE=
+
+# uncomment one of the below to load corresponding quantized model
+# LOAD_QUANTIZED_MODEL=4bit
+# LOAD_QUANTIZED_MODEL=8bit
+
+DISABLE_MODEL_PRELOADING=true
+CHAT_HISTORY_ENABLED=true
+SHOW_PARAM_SETTINGS=false
+SHARE_GRADIO_APP=false
+PDF_FILE_BASE_URL=https://chat-with-llama-2.netlify.app/pdfs/pci_dss_v4/
+
+# if unset, default to "hkunlp/instructor-xl"
+HF_EMBEDDINGS_MODEL_NAME="hkunlp/instructor-large"
+
+# number of cpu cores - used to set n_threads for GPT4ALL & LlamaCpp models
+NUMBER_OF_CPU_CORES=
+
+HUGGINGFACE_AUTH_TOKEN=
+
+USING_TORCH_BFLOAT16=true
+
+# HUGGINGFACE_MODEL_NAME_OR_PATH="databricks/dolly-v2-3b"
+# HUGGINGFACE_MODEL_NAME_OR_PATH="databricks/dolly-v2-7b"
+# HUGGINGFACE_MODEL_NAME_OR_PATH="databricks/dolly-v2-12b"
+# HUGGINGFACE_MODEL_NAME_OR_PATH="TheBloke/wizardLM-7B-HF"
+# HUGGINGFACE_MODEL_NAME_OR_PATH="TheBloke/vicuna-7B-1.1-HF"
+# HUGGINGFACE_MODEL_NAME_OR_PATH="nomic-ai/gpt4all-j"
+# HUGGINGFACE_MODEL_NAME_OR_PATH="nomic-ai/gpt4all-falcon"
+# HUGGINGFACE_MODEL_NAME_OR_PATH="lmsys/fastchat-t5-3b-v1.0"
+# HUGGINGFACE_MODEL_NAME_OR_PATH="meta-llama/Llama-2-7b-chat-hf"
+# HUGGINGFACE_MODEL_NAME_OR_PATH="meta-llama/Llama-2-13b-chat-hf"
+# HUGGINGFACE_MODEL_NAME_OR_PATH="meta-llama/Llama-2-70b-chat-hf"
+HUGGINGFACE_MODEL_NAME_OR_PATH="microsoft/Orca-2-7b"
+
+STABLELM_MODEL_NAME_OR_PATH="OpenAssistant/stablelm-7b-sft-v7-epoch-3"
+
+MOSAICML_MODEL_NAME_OR_PATH="mosaicml/mpt-7b-instruct"
+
+FALCON_MODEL_NAME_OR_PATH="tiiuae/falcon-7b-instruct"
+
+GPT4ALL_J_MODEL_PATH="./models/ggml-gpt4all-j-v1.3-groovy.bin"
+GPT4ALL_J_DOWNLOAD_LINK=https://gpt4all.io/models/ggml-gpt4all-j-v1.3-groovy.bin
+
+GPT4ALL_MODEL_PATH="./models/ggml-nous-gpt4-vicuna-13b.bin"
+GPT4ALL_DOWNLOAD_LINK=https://gpt4all.io/models/ggml-nous-gpt4-vicuna-13b.bin
+
+LLAMACPP_MODEL_PATH="./models/wizardLM-7B.ggmlv3.q4_1.bin"
+LLAMACPP_DOWNLOAD_LINK=https://huggingface.co/TheBloke/wizardLM-7B-GGML/resolve/main/wizardLM-7B.ggmlv3.q4_1.bin
+
+# Index for PCI DSS v4 PDF files - chunk_size=1024 chunk_overlap=512
+# CHROMADB_INDEX_PATH="./data/chromadb_1024_512/"
+FAISS_INDEX_PATH="./data/faiss_1024_512/"
+
+CHAT_QUESTION="What's the capital city of Malaysia?"
+QA_QUESTION="What's PCI DSS?"
+
+QUESTIONS_FILE_PATH="./data/questions.txt"
+
+TOKENIZERS_PARALLELISM=true
+
+# env variables for ingesting source PDF files
+SOURCE_PDFS_PATH="./data/pdfs/"
+SOURCE_URLS="./data/pci_dss_urls.txt"
+CHUNCK_SIZE=1024
+CHUNK_OVERLAP=512

.gitattributes

@@ -0,0 +1,39 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+data/faiss_1024_512/index.faiss filter=lfs diff=lfs merge=lfs -text
+data/faiss_1024_512/index.pkl filter=lfs diff=lfs merge=lfs -text
+data/chromadb_1024_512/9ccc1a9f-e406-4855-be51-a5d2aa027345 filter=lfs diff=lfs merge=lfs -text
+data/chromadb_1024_512/chroma.sqlite3 filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,3 +1,7 @@
+*.out
+pdfs/
+.vscode/
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -20,6 +24,7 @@ parts/
 sdist/
 var/
 wheels/
+pip-wheel-metadata/
 share/python-wheels/
 *.egg-info/
 .installed.cfg
@@ -49,14 +54,13 @@ coverage.xml
 *.py,cover
 .hypothesis/
 .pytest_cache/
-cover/
 
 # Translations
 *.mo
 *.pot
 
 # Django stuff:
-*.log
+# *.log
 local_settings.py
 db.sqlite3
 db.sqlite3-journal
@@ -72,7 +76,6 @@ instance/
 docs/_build/
 
 # PyBuilder
-.pybuilder/
 target/
 
 # Jupyter Notebook
@@ -83,9 +86,7 @@ profile_default/
 ipython_config.py
 
 # pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
+.python-version
 
 # pipenv
 #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
@@ -94,22 +95,7 @@ ipython_config.py
 #   install all needed dependencies.
 #Pipfile.lock
 
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
-# pdm
-#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#pdm.lock
-#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
-#   in version control.
-#   https://pdm.fming.dev/#use-with-ide
-.pdm.toml
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
 __pypackages__/
 
 # Celery stuff
@@ -146,15 +132,12 @@ dmypy.json
 # Pyre type checker
 .pyre/
 
-# pytype static type analyzer
-.pytype/
+# JetBrains
+.idea
+
+*.db
 
-# Cython debug symbols
-cython_debug/
+.DS_Store
 
-# PyCharm
-#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-#  and can be added to the global gitignore or merged into this file.  For a more nuclear
-#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+vectorstore.pkl
+langchain.readthedocs.io/

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "ragas_extended"]
+	path = ragas_extended
+	url = https://github.com/inflaton/ragas_extended.git

Makefile

@@ -0,0 +1,21 @@
+.PHONY: start
+start:
+	python app.py
+
+test:
+	python qa_chain_test.py
+
+ingest:
+	python ingest.py
+
+.PHONY: format
+format:
+	black .
+
+install:
+	pip install -r requirements.txt
+	cd ragas_extended && pip install -e .
+
+install-mac:
+	pip install -r requirements-mac.txt
+	cd ragas_extended && pip install -e .

README.md

@@ -1 +1,115 @@
-# chat-with-research-paper
+# ChatPDF - Talk to Your PDF Files 
+
+This project uses Open AI and open-source large language models (LLMs) to enable you to talk to your own PDF files.
+
+## How it works
+
+We're using an AI design pattern, namely "in-context learning" which uses LLMs off the shelf (i.e., without any fine-tuning), then controls their behavior through clever prompting and conditioning on private “contextual” data, e.g., texts extracted from your PDF files.
+
+At a very high level, the workflow can be divided into three stages:
+
+1. Data preprocessing / embedding: This stage involves storing private data (your PDF files) to be retrieved later. Typically, the documents are broken into chunks, passed through an embedding model, then stored the created embeddings in a vectorstore.
+
+2. Prompt construction / retrieval: When a user submits a query, the application constructs a series of prompts to submit to the language model. A compiled prompt typically combines a prompt template and a set of relevant documents retrieved from the vectorstore.
+
+3. Prompt execution / inference: Once the prompts have been compiled, they are submitted to a pre-trained LLM for inference—including both proprietary model APIs and open-source or self-trained models.
+
+![In-context Learning - Workflow Overview](./assets/Workflow-Overview.png)
+
+Tech stack used includes LangChain, Gradio, Chroma and FAISS. 
+- LangChain is an open-source framework that makes it easier to build scalable AI/LLM apps and chatbots. 
+- Gradio is an open-source Python library that is used to build machine learning and data science demos and web applications.
+- Chroma and FAISS are open-source vectorstores for storing embeddings for your files.
+
+## Running Locally
+
+1. Check pre-conditions:
+
+- Run `python --version` to make sure you're running Python version 3.10 or above.
+- The latest PyTorch with GPU support must have been installed. Here is a sample `conda` command:
+```
+conda install -y pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch -c nvidia
+```
+- [CMake](https://cmake.org/) must have been installed. Here is a sample command to install `CMake` on `ubuntu`:
+```
+sudo apt install cmake
+```
+
+2. Clone the repo
+
+```
+git clone https://github.com/smu-ai/Evaluation-of-Orca-2-Models-for-Conversational-RAG.git
+```
+
+
+3. Install packages
+
+On Linux/WSL2:
+```
+pip install -r requirements.txt
+```
+
+On Mac:
+```
+pip install -r requirements-mac.txt
+```
+
+4. Set up your environment variables
+
+- By default, environment variables are loaded from `.env.example` file
+- If you don't want to use the default settings, copy `.env.example` into `.env`. Your can then update it for your local runs.
+
+
+5. Start the local server at `http://localhost:7860`:
+
+```
+python app.py
+```
+
+## Talk to Your Own PDF Files
+
+- The sample PDF files are downloaded from [PCI DSS official website](https://www.pcisecuritystandards.org/document_library/?category=pcidss) and the corresponding embeddings are stored in folder `data/faiss_1024_512` with FAISS format, which allows you to run locally without any additional effort.
+
+- You can also put your own PDF files into any folder specified in `SOURCE_PDFS_PATH` and run the command below to generate embeddings which will be stored in folder `FAISS_INDEX_PATH` or `CHROMADB_INDEX_PATH`. If both `*_INDEX_PATH` env vars are set, `FAISS_INDEX_PATH` takes precedence. Make sure the folder specified by `*_INDEX_PATH` doesn't exist; other wise the command will simply try to load index from the folder and do a simple similarity search, as a way to verify if embeddings are generated and stored properly. Please note the HuggingFace Embedding model specified by `HF_EMBEDDINGS_MODEL_NAME` will be used to generate the embeddings.
+
+```
+python ingest.py
+```
+
+- Once embeddings are generated, you can test them out locally, or check them into your duplicated space. Please note HF Spaces git server does not allow PDF files to be checked in.
+
+## Play with Different Large Language Models
+
+The source code supports different LLM types - as shown at the top of `.env.example`
+
+```
+# LLM_MODEL_TYPE=openai
+# LLM_MODEL_TYPE=gpt4all-j
+# LLM_MODEL_TYPE=gpt4all
+# LLM_MODEL_TYPE=llamacpp
+LLM_MODEL_TYPE=huggingface
+# LLM_MODEL_TYPE=mosaicml
+# LLM_MODEL_TYPE=stablelm
+# LLM_MODEL_TYPE=openllm
+# LLM_MODEL_TYPE=hftgi
+```
+
+- By default, the app runs `lmsys/fastchat-t5-3b-v1.0` model with HF Transformers, which works well with most PCs/laptops with 32GB or more RAM, without any GPU. It also works on HF Spaces with their free-tier: 2 vCPU, 16GB RAM and 500GB hard disk, though the inference speed is very slow.
+
+- Uncomment/comment the above to play with different LLM types. You may also want to update other related env vars. E.g., here's the list of HF models which have been tested with the code:
+
+```
+# HUGGINGFACE_MODEL_NAME_OR_PATH="databricks/dolly-v2-3b"
+# HUGGINGFACE_MODEL_NAME_OR_PATH="databricks/dolly-v2-7b"
+# HUGGINGFACE_MODEL_NAME_OR_PATH="databricks/dolly-v2-12b"
+# HUGGINGFACE_MODEL_NAME_OR_PATH="TheBloke/wizardLM-7B-HF"
+# HUGGINGFACE_MODEL_NAME_OR_PATH="TheBloke/vicuna-7B-1.1-HF"
+# HUGGINGFACE_MODEL_NAME_OR_PATH="nomic-ai/gpt4all-j"
+# HUGGINGFACE_MODEL_NAME_OR_PATH="nomic-ai/gpt4all-falcon"
+HUGGINGFACE_MODEL_NAME_OR_PATH="lmsys/fastchat-t5-3b-v1.0"
+# HUGGINGFACE_MODEL_NAME_OR_PATH="meta-llama/Llama-2-7b-chat-hf"
+# HUGGINGFACE_MODEL_NAME_OR_PATH="meta-llama/Llama-2-13b-chat-hf"
+# HUGGINGFACE_MODEL_NAME_OR_PATH="meta-llama/Llama-2-70b-chat-hf"
+```
+
+The script `test.sh` automates running different LLMs and records the outputs in `data/logs` folder which currently contains a few log files created by previous test runs on Nvidia GeForce RTX 4090, A40 and L40 GPUs.

app.py

@@ -0,0 +1,203 @@
+"""Main entrypoint for the app."""
+import os
+import time
+from queue import Queue
+from timeit import default_timer as timer
+
+import gradio as gr
+from anyio.from_thread import start_blocking_portal
+
+from app_modules.init import app_init
+from app_modules.utils import print_llm_response, remove_extra_spaces
+
+llm_loader, qa_chain = app_init()
+
+show_param_settings = os.environ.get("SHOW_PARAM_SETTINGS") == "true"
+share_gradio_app = os.environ.get("SHARE_GRADIO_APP") == "true"
+using_openai = os.environ.get("LLM_MODEL_TYPE") == "openai"
+chat_history_enabled = os.environ.get("CHAT_HISTORY_ENABLED") == "true"
+
+model = (
+    "OpenAI GPT-3.5"
+    if using_openai
+    else os.environ.get("HUGGINGFACE_MODEL_NAME_OR_PATH")
+)
+href = (
+    "https://platform.openai.com/docs/models/gpt-3-5"
+    if using_openai
+    else f"https://huggingface.co/{model}"
+)
+
+name = "PCI DSS v4"
+
+title = f"""<h1 align="left" style="min-width:200px; margin-top:0;"> Chat with {name} </h1>"""
+
+description_top = f"""\
+<div align="left">
+<p> Currently Running: <a href="{href}">{model}</a></p>
+</div>
+"""
+
+description = """\
+<div align="center" style="margin:16px 0">
+The demo is built on <a href="https://github.com/hwchase17/langchain">LangChain</a>.
+</div>
+"""
+
+CONCURRENT_COUNT = 1
+
+
+def qa(chatbot):
+    user_msg = chatbot[-1][0]
+    q = Queue()
+    result = Queue()
+    job_done = object()
+
+    def task(question, chat_history):
+        start = timer()
+        inputs = {"question": question}
+        inputs["chat_history"] = chat_history
+        ret = qa_chain.call_chain(inputs, None, q)
+        end = timer()
+
+        print(f"Completed in {end - start:.3f}s")
+        print_llm_response(ret)
+
+        q.put(job_done)
+        result.put(ret)
+
+    with start_blocking_portal() as portal:
+        chat_history = []
+        if chat_history_enabled:
+            for i in range(len(chatbot) - 1):
+                element = chatbot[i]
+                item = (element[0] or "", element[1] or "")
+                chat_history.append(item)
+
+        portal.start_task_soon(task, user_msg, chat_history)
+
+        content = ""
+        count = 2 if len(chat_history) > 0 else 1
+
+        while count > 0:
+            while q.empty():
+                print("nothing generated yet - retry in 0.5s")
+                time.sleep(0.5)
+
+            for next_token in llm_loader.streamer:
+                if next_token is job_done:
+                    break
+                content += next_token or ""
+                chatbot[-1][1] = remove_extra_spaces(content)
+
+                if count == 1:
+                    yield chatbot
+
+            count -= 1
+
+        chatbot[-1][1] += "\n\nSources:\n"
+        ret = result.get()
+        titles = []
+        for doc in ret["source_documents"]:
+            page = doc.metadata["page"] + 1
+            url = f"{doc.metadata['url']}#page={page}"
+            file_name = doc.metadata["source"].split("/")[-1]
+            title = f"{file_name} Page: {page}"
+            if title not in titles:
+                titles.append(title)
+                chatbot[-1][1] += f"1. [{title}]({url})\n"
+
+        yield chatbot
+
+
+with open("assets/custom.css", "r", encoding="utf-8") as f:
+    customCSS = f.read()
+
+with gr.Blocks(css=customCSS) as demo:
+    user_question = gr.State("")
+    with gr.Row():
+        gr.HTML(title)
+    gr.Markdown(description_top)
+    with gr.Row(equal_height=True):
+        with gr.Column(scale=5):
+            with gr.Row():
+                chatbot = gr.Chatbot(elem_id="inflaton_chatbot", height="100%")
+            with gr.Row():
+                with gr.Column(scale=2):
+                    user_input = gr.Textbox(
+                        show_label=False,
+                        placeholder="Enter your question here",
+                        container=False,
+                    )
+                with gr.Column(
+                    min_width=70,
+                ):
+                    submitBtn = gr.Button("Send")
+                with gr.Column(
+                    min_width=70,
+                ):
+                    clearBtn = gr.Button("Clear")
+        if show_param_settings:
+            with gr.Column():
+                with gr.Column(
+                    min_width=50,
+                ):
+                    with gr.Tab(label="Parameter Setting"):
+                        gr.Markdown("# Parameters")
+                        top_p = gr.Slider(
+                            minimum=-0,
+                            maximum=1.0,
+                            value=0.95,
+                            step=0.05,
+                            # interactive=True,
+                            label="Top-p",
+                        )
+                        temperature = gr.Slider(
+                            minimum=0.1,
+                            maximum=2.0,
+                            value=0,
+                            step=0.1,
+                            # interactive=True,
+                            label="Temperature",
+                        )
+                        max_new_tokens = gr.Slider(
+                            minimum=0,
+                            maximum=2048,
+                            value=2048,
+                            step=8,
+                            # interactive=True,
+                            label="Max Generation Tokens",
+                        )
+                        max_context_length_tokens = gr.Slider(
+                            minimum=0,
+                            maximum=4096,
+                            value=4096,
+                            step=128,
+                            # interactive=True,
+                            label="Max Context Tokens",
+                        )
+    gr.Markdown(description)
+
+    def chat(user_message, history):
+        return "", history + [[user_message, None]]
+
+    user_input.submit(
+        chat, [user_input, chatbot], [user_input, chatbot], queue=True
+    ).then(qa, chatbot, chatbot)
+
+    submitBtn.click(
+        chat, [user_input, chatbot], [user_input, chatbot], queue=True, api_name="chat"
+    ).then(qa, chatbot, chatbot)
+
+    def reset():
+        return "", []
+
+    clearBtn.click(
+        reset,
+        outputs=[user_input, chatbot],
+        show_progress=True,
+        api_name="reset",
+    )
+
+demo.title = "Chat with PCI DSS v4"
+demo.queue().launch(share=share_gradio_app)

app_modules/init.py

@@ -0,0 +1,83 @@
+"""Main entrypoint for the app."""
+import os
+from timeit import default_timer as timer
+from typing import List, Optional
+
+from dotenv import find_dotenv, load_dotenv
+from langchain_community.embeddings import HuggingFaceInstructEmbeddings
+from langchain.vectorstores.chroma import Chroma
+from langchain.vectorstores.faiss import FAISS
+
+from app_modules.llm_loader import LLMLoader
+from app_modules.llm_qa_chain import QAChain
+from app_modules.utils import get_device_types, init_settings
+
+found_dotenv = find_dotenv(".env")
+
+if len(found_dotenv) == 0:
+    found_dotenv = find_dotenv(".env.example")
+print(f"loading env vars from: {found_dotenv}")
+load_dotenv(found_dotenv, override=False)
+
+# Constants
+init_settings()
+
+if os.environ.get("LANGCHAIN_DEBUG") == "true":
+    import langchain
+
+    langchain.debug = True
+
+
+def app_init():
+    # https://github.com/huggingface/transformers/issues/17611
+    os.environ["CURL_CA_BUNDLE"] = ""
+
+    hf_embeddings_device_type, hf_pipeline_device_type = get_device_types()
+    print(f"hf_embeddings_device_type: {hf_embeddings_device_type}")
+    print(f"hf_pipeline_device_type: {hf_pipeline_device_type}")
+
+    hf_embeddings_model_name = (
+        os.environ.get("HF_EMBEDDINGS_MODEL_NAME") or "hkunlp/instructor-xl"
+    )
+
+    n_threds = int(os.environ.get("NUMBER_OF_CPU_CORES") or "4")
+    index_path = os.environ.get("FAISS_INDEX_PATH") or os.environ.get(
+        "CHROMADB_INDEX_PATH"
+    )
+    using_faiss = os.environ.get("FAISS_INDEX_PATH") is not None
+    llm_model_type = os.environ.get("LLM_MODEL_TYPE")
+
+    start = timer()
+    embeddings = HuggingFaceInstructEmbeddings(
+        model_name=hf_embeddings_model_name,
+        model_kwargs={"device": hf_embeddings_device_type},
+    )
+    end = timer()
+
+    print(f"Completed in {end - start:.3f}s")
+
+    start = timer()
+
+    print(f"Load index from {index_path} with {'FAISS' if using_faiss else 'Chroma'}")
+
+    if not os.path.isdir(index_path):
+        raise ValueError(f"{index_path} does not exist!")
+    elif using_faiss:
+        vectorstore = FAISS.load_local(index_path, embeddings)
+    else:
+        vectorstore = Chroma(
+            embedding_function=embeddings, persist_directory=index_path
+        )
+
+    end = timer()
+
+    print(f"Completed in {end - start:.3f}s")
+
+    start = timer()
+    llm_loader = LLMLoader(llm_model_type)
+    llm_loader.init(n_threds=n_threds, hf_pipeline_device_type=hf_pipeline_device_type)
+    qa_chain = QAChain(vectorstore, llm_loader)
+    end = timer()
+    print(f"Completed in {end - start:.3f}s")
+
+    return llm_loader, qa_chain

app_modules/llm_inference.py

@@ -0,0 +1,137 @@
+import abc
+import os
+import time
+import urllib
+from queue import Queue
+from threading import Thread
+from typing import List, Optional
+from urllib.parse import quote, urlparse, urlunparse
+
+from langchain.chains.base import Chain
+
+from app_modules.llm_loader import LLMLoader, TextIteratorStreamer
+from app_modules.utils import remove_extra_spaces
+
+
+class LLMInference(metaclass=abc.ABCMeta):
+    def __init__(self, llm_loader):
+        self.llm_loader = llm_loader
+        self.chain = None
+
+    @abc.abstractmethod
+    def create_chain(self) -> Chain:
+        pass
+
+    def get_chain(self) -> Chain:
+        if self.chain is None:
+            self.chain = self.create_chain()
+
+        return self.chain
+
+    def run_chain(self, chain, inputs, callbacks: Optional[List] = []):
+        result = chain.invoke(inputs, {"callbacks": callbacks})
+        if "text" in result:
+            result["response"] = result["text"]
+            del result["text"]
+        return result
+
+    def call_chain(
+        self,
+        inputs,
+        streaming_handler,
+        q: Queue = None,
+        testing: bool = False,
+    ):
+        print(inputs)
+        if self.llm_loader.streamer.for_huggingface:
+            self.llm_loader.lock.acquire()
+
+        try:
+            self.llm_loader.streamer.reset(q)
+
+            chain = self.get_chain()
+            result = (
+                self._run_chain_with_streaming_handler(
+                    chain, inputs, streaming_handler, testing
+                )
+                if streaming_handler is not None
+                else self.run_chain(chain, inputs)
+            )
+
+            if "answer" in result:
+                result["answer"] = remove_extra_spaces(result["answer"])
+
+                source_path = os.environ.get("SOURCE_PATH")
+                base_url = os.environ.get("PDF_FILE_BASE_URL")
+                if base_url is not None and len(base_url) > 0:
+                    documents = result["source_documents"]
+                    for doc in documents:
+                        source = doc.metadata["source"]
+                        title = source.split("/")[-1]
+                        doc.metadata["url"] = f"{base_url}{urllib.parse.quote(title)}"
+                elif source_path is not None and len(source_path) > 0:
+                    documents = result["source_documents"]
+                    for doc in documents:
+                        source = doc.metadata["source"]
+                        url = source.replace(source_path, "https://")
+                        url = url.replace(".html", "")
+                        parsed_url = urlparse(url)
+
+                        # Encode path, query, and fragment
+                        encoded_path = quote(parsed_url.path)
+                        encoded_query = quote(parsed_url.query)
+                        encoded_fragment = quote(parsed_url.fragment)
+
+                        # Construct the encoded URL
+                        doc.metadata["url"] = urlunparse(
+                            (
+                                parsed_url.scheme,
+                                parsed_url.netloc,
+                                encoded_path,
+                                parsed_url.params,
+                                encoded_query,
+                                encoded_fragment,
+                            )
+                        )
+
+            return result
+        finally:
+            if self.llm_loader.streamer.for_huggingface:
+                self.llm_loader.lock.release()
+
+    def _execute_chain(self, chain, inputs, q, sh):
+        q.put(self.run_chain(chain, inputs, callbacks=[sh]))
+
+    def _run_chain_with_streaming_handler(
+        self, chain, inputs, streaming_handler, testing
+    ):
+        que = Queue()
+
+        t = Thread(
+            target=self._execute_chain,
+            args=(chain, inputs, que, streaming_handler),
+        )
+        t.start()
+
+        if self.llm_loader.streamer.for_huggingface:
+            count = (
+                2
+                if "chat_history" in inputs and len(inputs.get("chat_history")) > 0
+                else 1
+            )
+
+            while count > 0:
+                try:
+                    for token in self.llm_loader.streamer:
+                        if not testing:
+                            streaming_handler.on_llm_new_token(token)
+
+                    self.llm_loader.streamer.reset()
+                    count -= 1
+                except Exception:
+                    if not testing:
+                        print("nothing generated yet - retry in 0.5s")
+                    time.sleep(0.5)
+
+        t.join()
+        return que.get()

app_modules/llm_loader.py

@@ -0,0 +1,648 @@
+import os
+import sys
+import threading
+from queue import Queue
+from typing import Any, Optional
+
+import torch
+from langchain.callbacks.base import BaseCallbackHandler
+from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
+from langchain_openai.chat_models import ChatOpenAI
+from langchain_openai.llms import OpenAI
+from langchain_community.llms import (
+    HuggingFaceTextGenInference,
+    CTransformers,
+    GPT4All,
+    HuggingFacePipeline,
+    LlamaCpp,
+)
+from langchain.schema import LLMResult
+from transformers import (
+    AutoConfig,
+    AutoModelForCausalLM,
+    AutoModelForSeq2SeqLM,
+    AutoTokenizer,
+    BitsAndBytesConfig,
+    StoppingCriteria,
+    StoppingCriteriaList,
+    T5Tokenizer,
+    TextStreamer,
+    pipeline,
+)
+
+from app_modules.utils import ensure_model_is_downloaded
+
+
+class TextIteratorStreamer(TextStreamer, StreamingStdOutCallbackHandler):
+    def __init__(
+        self,
+        tokenizer: "AutoTokenizer",
+        skip_prompt: bool = False,
+        timeout: Optional[float] = None,
+        for_huggingface: bool = False,
+        **decode_kwargs,
+    ):
+        super().__init__(tokenizer, skip_prompt, **decode_kwargs)
+        self.text_queue = Queue()
+        self.stop_signal = None
+        self.timeout = timeout
+        self.total_tokens = 0
+        self.for_huggingface = for_huggingface
+
+    def on_finalized_text(self, text: str, stream_end: bool = False):
+        super().on_finalized_text(text, stream_end=stream_end)
+
+        """Put the new text in the queue. If the stream is ending, also put a stop signal in the queue."""
+        self.text_queue.put(text, timeout=self.timeout)
+        self.total_tokens = self.total_tokens + 1
+        if stream_end:
+            print("\n")
+            self.text_queue.put("\n", timeout=self.timeout)
+            self.text_queue.put(self.stop_signal, timeout=self.timeout)
+
+    def on_llm_new_token(self, token: str, **kwargs: Any) -> None:
+        sys.stdout.write(token)
+        sys.stdout.flush()
+        self.text_queue.put(token, timeout=self.timeout)
+        self.total_tokens = self.total_tokens + 1
+
+    def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None:
+        print("\n")
+        self.text_queue.put("\n", timeout=self.timeout)
+        self.text_queue.put(self.stop_signal, timeout=self.timeout)
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        value = self.text_queue.get(timeout=self.timeout)
+        if value == self.stop_signal:
+            raise StopIteration()
+        else:
+            return value
+
+    def reset(self, q: Queue = None):
+        # print("resetting TextIteratorStreamer")
+        self.text_queue = q if q is not None else Queue()
+
+    def empty(self):
+        return self.text_queue.empty()
+
+
+class LLMLoader:
+    def __init__(self, llm_model_type):
+        self.llm_model_type = llm_model_type
+        self.llm = None
+        self.streamer = TextIteratorStreamer(
+            "",
+            for_huggingface=True,
+        )
+        self.max_tokens_limit = 2048
+        self.search_kwargs = {"k": 4}
+        self.lock = threading.Lock()
+
+    def _init_hf_streamer(self, tokenizer):
+        self.streamer = TextIteratorStreamer(
+            tokenizer,
+            timeout=10.0,
+            skip_prompt=True,
+            skip_special_tokens=True,
+            for_huggingface=True,
+        )
+
+    def init(
+        self,
+        custom_handler: Optional[BaseCallbackHandler] = None,
+        n_threds: int = 4,
+        hf_pipeline_device_type: str = None,
+    ):
+        print("initializing LLM: " + self.llm_model_type)
+
+        if hf_pipeline_device_type is None:
+            hf_pipeline_device_type = "cpu"
+
+        using_cuda = hf_pipeline_device_type.startswith("cuda")
+        torch_dtype = torch.float16 if using_cuda else torch.float32
+        if os.environ.get("USING_TORCH_BFLOAT16") == "true":
+            torch_dtype = torch.bfloat16
+        load_quantized_model = os.environ.get("LOAD_QUANTIZED_MODEL")
+
+        print(f"  hf_pipeline_device_type: {hf_pipeline_device_type}")
+        print(f"     load_quantized_model: {load_quantized_model}")
+        print(f"              torch_dtype: {torch_dtype}")
+        print(f"                 n_threds: {n_threds}")
+
+        torch.set_default_dtype(torch_dtype)
+
+        double_quant_config = BitsAndBytesConfig(
+            load_in_4bit=load_quantized_model == "4bit",
+            bnb_4bit_use_double_quant=load_quantized_model == "4bit",
+            load_in_8bit=load_quantized_model == "8bit",
+            bnb_8bit_use_double_quant=load_quantized_model == "8bit",
+        )
+
+        callbacks = []
+        if self.streamer is not None:
+            callbacks.append(self.streamer)
+        if custom_handler is not None:
+            callbacks.append(custom_handler)
+
+        if self.llm is None:
+            if self.llm_model_type == "openai":
+                MODEL_NAME = os.environ.get("OPENAI_MODEL_NAME") or "gpt-3.5-turbo"
+                print(f"              using model: {MODEL_NAME}")
+                self.llm = (
+                    OpenAI(
+                        model_name=MODEL_NAME,
+                        streaming=True,
+                        callbacks=callbacks,
+                        verbose=True,
+                        temperature=0,
+                    )
+                    if "instruct" in MODEL_NAME
+                    else ChatOpenAI(
+                        model_name=MODEL_NAME,
+                        streaming=True,
+                        callbacks=callbacks,
+                        verbose=True,
+                        temperature=0,
+                    )
+                )
+            elif self.llm_model_type.startswith("gpt4all"):
+                MODEL_PATH = ensure_model_is_downloaded(self.llm_model_type)
+                self.llm = GPT4All(
+                    model=MODEL_PATH,
+                    max_tokens=2048,
+                    n_threads=n_threds,
+                    backend="gptj" if self.llm_model_type == "gpt4all-j" else "llama",
+                    callbacks=callbacks,
+                    verbose=True,
+                    use_mlock=True,
+                )
+            elif self.llm_model_type == "llamacpp":
+                MODEL_PATH = ensure_model_is_downloaded(self.llm_model_type)
+                self.llm = LlamaCpp(
+                    model_path=MODEL_PATH,
+                    n_ctx=8192,
+                    n_threads=n_threds,
+                    seed=0,
+                    temperature=0,
+                    max_tokens=2048,
+                    callbacks=callbacks,
+                    verbose=True,
+                    use_mlock=True,
+                )
+            elif self.llm_model_type == "ctransformers":
+                MODEL_PATH = ensure_model_is_downloaded(self.llm_model_type)
+                config = {
+                    "max_new_tokens": self.max_tokens_limit,
+                    "repetition_penalty": 1.1,
+                }
+                self.llm = CTransformers(
+                    model=MODEL_PATH,
+                    model_type="llama",
+                    config=config,
+                    callbacks=callbacks,
+                )
+            elif self.llm_model_type == "hftgi":
+                HFTGI_SERVER_URL = os.environ.get("HFTGI_SERVER_URL")
+                HFTGI_RP = os.environ.get("HFTGI_RP")
+                repetition_penalty = 1.120 if HFTGI_RP is None else float(HFTGI_RP)
+                print(f"       repetition_penalty: {repetition_penalty}")
+                self.max_tokens_limit = 4096
+                self.llm = HuggingFaceTextGenInference(
+                    inference_server_url=HFTGI_SERVER_URL,
+                    max_new_tokens=self.max_tokens_limit / 2,
+                    # top_k=0,
+                    top_p=0.95,
+                    # typical_p=0.95,
+                    temperature=0.01,
+                    repetition_penalty=repetition_penalty,
+                    callbacks=callbacks,
+                    timeout=600,
+                    streaming=True,
+                )
+            elif self.llm_model_type.startswith("huggingface"):
+                MODEL_NAME_OR_PATH = os.environ.get("HUGGINGFACE_MODEL_NAME_OR_PATH")
+                print(f"            loading model: {MODEL_NAME_OR_PATH}")
+
+                hf_auth_token = (
+                    os.environ.get("HUGGINGFACE_AUTH_TOKEN")
+                    if "Llama-2" in MODEL_NAME_OR_PATH
+                    else None
+                )
+                transformers_offline = os.environ.get("TRANSFORMERS_OFFLINE") == "1"
+                token = (
+                    hf_auth_token
+                    if hf_auth_token is not None
+                    and len(hf_auth_token) > 0
+                    and not transformers_offline
+                    else None
+                )
+                print(f"            HF auth token: {str(token)[-5:]}")
+
+                if "Llama-2" in MODEL_NAME_OR_PATH:
+                    self.max_tokens_limit = 4096
+
+                is_t5 = "t5" in MODEL_NAME_OR_PATH
+                temperature = (
+                    0.01
+                    if "gpt4all-j" in MODEL_NAME_OR_PATH
+                    or "dolly" in MODEL_NAME_OR_PATH
+                    or "Qwen" in MODEL_NAME_OR_PATH
+                    or "Llama-2" in MODEL_NAME_OR_PATH
+                    or "Orca-2" in MODEL_NAME_OR_PATH
+                    or "phi-2" in MODEL_NAME_OR_PATH
+                    else 0
+                )
+                use_fast = (
+                    "stable" in MODEL_NAME_OR_PATH
+                    or "RedPajama" in MODEL_NAME_OR_PATH
+                    or "dolly" in MODEL_NAME_OR_PATH
+                )
+                padding_side = "left"  # if "dolly" in MODEL_NAME_OR_PATH else None
+
+                config = (
+                    AutoConfig.from_pretrained(
+                        MODEL_NAME_OR_PATH,
+                        trust_remote_code=True,
+                        token=token,
+                        fp32=hf_pipeline_device_type == "cpu",
+                        bf16=(
+                            hf_pipeline_device_type != "cpu"
+                            and torch_dtype == torch.bfloat16
+                        ),
+                        fp16=(
+                            hf_pipeline_device_type != "cpu"
+                            and torch_dtype != torch.bfloat16
+                        ),
+                    )
+                    if "Qwen" in MODEL_NAME_OR_PATH
+                    else AutoConfig.from_pretrained(
+                        MODEL_NAME_OR_PATH,
+                        trust_remote_code=True,
+                        token=token,
+                    )
+                )
+
+                # config.attn_config["attn_impl"] = "triton"
+                # config.max_seq_len = 4096
+                config.init_device = hf_pipeline_device_type
+
+                tokenizer = (
+                    T5Tokenizer.from_pretrained(
+                        MODEL_NAME_OR_PATH,
+                        token=token,
+                    )
+                    if is_t5
+                    else AutoTokenizer.from_pretrained(
+                        MODEL_NAME_OR_PATH,
+                        use_fast=use_fast,
+                        trust_remote_code=True,
+                        padding_side=padding_side,
+                        token=token,
+                    )
+                )
+
+                self._init_hf_streamer(tokenizer)
+
+                task = "text2text-generation" if is_t5 else "text-generation"
+
+                return_full_text = True if "dolly" in MODEL_NAME_OR_PATH else None
+
+                repetition_penalty = (
+                    1.15
+                    if "falcon" in MODEL_NAME_OR_PATH
+                    else (1.25 if "dolly" in MODEL_NAME_OR_PATH else 1.1)
+                )
+
+                HF_RP = os.environ.get("HF_RP")
+                if HF_RP is not None and len(HF_RP) > 0:
+                    repetition_penalty = float(HF_RP)
+                print(f"       repetition_penalty: {repetition_penalty}")
+
+                if load_quantized_model is not None:
+                    model = (
+                        AutoModelForSeq2SeqLM.from_pretrained(
+                            MODEL_NAME_OR_PATH,
+                            config=config,
+                            quantization_config=double_quant_config,
+                            trust_remote_code=True,
+                            use_auth_token=token,
+                        )
+                        if is_t5
+                        else AutoModelForCausalLM.from_pretrained(
+                            MODEL_NAME_OR_PATH,
+                            config=config,
+                            quantization_config=double_quant_config,
+                            trust_remote_code=True,
+                            use_auth_token=token,
+                        )
+                    )
+
+                    print(f"Model memory footprint: {model.get_memory_footprint()}")
+
+                    eos_token_id = -1
+                    # starchat-beta uses a special <|end|> token with ID 49155 to denote ends of a turn
+                    if "starchat" in MODEL_NAME_OR_PATH:
+                        eos_token_id = 49155
+                    pad_token_id = eos_token_id
+
+                    pipe = (
+                        pipeline(
+                            task,
+                            model=model,
+                            tokenizer=tokenizer,
+                            eos_token_id=eos_token_id,
+                            pad_token_id=pad_token_id,
+                            streamer=self.streamer,
+                            return_full_text=return_full_text,  # langchain expects the full text
+                            device_map="auto",
+                            trust_remote_code=True,
+                            max_new_tokens=2048,
+                            do_sample=True,
+                            temperature=0.01,
+                            top_p=0.95,
+                            top_k=50,
+                            repetition_penalty=repetition_penalty,
+                        )
+                        if eos_token_id != -1
+                        else pipeline(
+                            task,
+                            model=model,
+                            tokenizer=tokenizer,
+                            streamer=self.streamer,
+                            return_full_text=return_full_text,  # langchain expects the full text
+                            device_map="auto",
+                            trust_remote_code=True,
+                            max_new_tokens=2048,
+                            do_sample=True,
+                            temperature=temperature,
+                            top_p=0.95,
+                            top_k=0,  # select from top 0 tokens (because zero, relies on top_p)
+                            repetition_penalty=repetition_penalty,
+                        )
+                    )
+                else:
+                    if os.environ.get("DISABLE_MODEL_PRELOADING") != "true":
+                        model = (
+                            AutoModelForSeq2SeqLM.from_pretrained(
+                                MODEL_NAME_OR_PATH,
+                                config=config,
+                                trust_remote_code=True,
+                            )
+                            if is_t5
+                            else (
+                                AutoModelForCausalLM.from_pretrained(
+                                    MODEL_NAME_OR_PATH,
+                                    config=config,
+                                    trust_remote_code=True,
+                                )
+                                if "Qwen" in MODEL_NAME_OR_PATH
+                                else (
+                                    AutoModelForCausalLM.from_pretrained(
+                                        MODEL_NAME_OR_PATH,
+                                        config=config,
+                                        trust_remote_code=True,
+                                    )
+                                    if token is None
+                                    else AutoModelForCausalLM.from_pretrained(
+                                        MODEL_NAME_OR_PATH,
+                                        config=config,
+                                        trust_remote_code=True,
+                                        token=token,
+                                    )
+                                )
+                            )
+                        )
+                        print(f"Model memory footprint: {model.get_memory_footprint()}")
+                        model = model.eval()
+                        # print(f"Model memory footprint: {model.get_memory_footprint()}")
+                    else:
+                        model = MODEL_NAME_OR_PATH
+
+                    pipe = (
+                        pipeline(
+                            task,
+                            model=model,
+                            tokenizer=tokenizer,
+                            streamer=self.streamer,
+                            return_full_text=return_full_text,  # langchain expects the full text
+                            device=hf_pipeline_device_type,
+                            torch_dtype=torch_dtype,
+                            max_new_tokens=2048,
+                            trust_remote_code=True,
+                            do_sample=True,
+                            temperature=temperature,
+                            top_p=0.95,
+                            top_k=0,  # select from top 0 tokens (because zero, relies on top_p)
+                            repetition_penalty=repetition_penalty,
+                        )
+                        if token is None
+                        else pipeline(
+                            task,
+                            model=model,
+                            tokenizer=tokenizer,
+                            streamer=self.streamer,
+                            return_full_text=return_full_text,  # langchain expects the full text
+                            device=hf_pipeline_device_type,
+                            torch_dtype=torch_dtype,
+                            max_new_tokens=2048,
+                            trust_remote_code=True,
+                            temperature=temperature,
+                            top_p=0.95,
+                            top_k=0,  # select from top 0 tokens (because zero, relies on top_p)
+                            repetition_penalty=repetition_penalty,
+                            token=token,
+                        )
+                    )
+
+                pipe.model.config.pad_token_id = pipe.model.config.eos_token_id
+                self.llm = HuggingFacePipeline(pipeline=pipe, callbacks=callbacks)
+            elif self.llm_model_type == "mosaicml":
+                MODEL_NAME_OR_PATH = os.environ.get("MOSAICML_MODEL_NAME_OR_PATH")
+                print(f"            loading model: {MODEL_NAME_OR_PATH}")
+
+                config = AutoConfig.from_pretrained(
+                    MODEL_NAME_OR_PATH, trust_remote_code=True
+                )
+                # config.attn_config["attn_impl"] = "triton"
+                config.max_seq_len = 16384 if "30b" in MODEL_NAME_OR_PATH else 4096
+                config.init_device = hf_pipeline_device_type
+
+                model = (
+                    AutoModelForCausalLM.from_pretrained(
+                        MODEL_NAME_OR_PATH,
+                        config=config,
+                        quantization_config=double_quant_config,
+                        trust_remote_code=True,
+                    )
+                    if load_quantized_model is not None
+                    else AutoModelForCausalLM.from_pretrained(
+                        MODEL_NAME_OR_PATH,
+                        config=config,
+                        torch_dtype=torch_dtype,
+                        trust_remote_code=True,
+                    )
+                )
+
+                print(f"Model loaded on {config.init_device}")
+                print(f"Model memory footprint: {model.get_memory_footprint()}")
+
+                tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b")
+                self._init_hf_streamer(tokenizer)
+
+                # mtp-7b is trained to add "<|endoftext|>" at the end of generations
+                stop_token_ids = tokenizer.convert_tokens_to_ids(["<|endoftext|>"])
+
+                # define custom stopping criteria object
+                class StopOnTokens(StoppingCriteria):
+                    def __call__(
+                        self,
+                        input_ids: torch.LongTensor,
+                        scores: torch.FloatTensor,
+                        **kwargs,
+                    ) -> bool:
+                        for stop_id in stop_token_ids:
+                            if input_ids[0][-1] == stop_id:
+                                return True
+                        return False
+
+                stopping_criteria = StoppingCriteriaList([StopOnTokens()])
+
+                max_new_tokens = 8192 if "30b" in MODEL_NAME_OR_PATH else 2048
+                self.max_tokens_limit = max_new_tokens
+                repetition_penalty = 1.05 if "30b" in MODEL_NAME_OR_PATH else 1.02
+
+                ML_RP = os.environ.get("ML_RP")
+                if ML_RP is not None and len(ML_RP) > 0:
+                    repetition_penalty = float(ML_RP)
+                print(f"       repetition_penalty: {repetition_penalty}")
+
+                pipe = (
+                    pipeline(
+                        model=model,
+                        tokenizer=tokenizer,
+                        streamer=self.streamer,
+                        return_full_text=True,  # langchain expects the full text
+                        task="text-generation",
+                        device_map="auto",
+                        # we pass model parameters here too
+                        stopping_criteria=stopping_criteria,  # without this model will ramble
+                        do_sample=True,
+                        temperature=0.01,  # 'randomness' of outputs, 0.0 is the min and 1.0 the max
+                        top_p=0.95,  # select from top tokens whose probability add up to 15%
+                        top_k=0,  # select from top 0 tokens (because zero, relies on top_p)
+                        max_new_tokens=max_new_tokens,  # mex number of tokens to generate in the output
+                        repetition_penalty=repetition_penalty,  # without this output begins repeating
+                    )
+                    if load_quantized_model is not None
+                    else pipeline(
+                        model=model,
+                        tokenizer=tokenizer,
+                        streamer=self.streamer,
+                        return_full_text=True,  # langchain expects the full text
+                        task="text-generation",
+                        device=config.init_device,
+                        # we pass model parameters here too
+                        stopping_criteria=stopping_criteria,  # without this model will ramble
+                        do_sample=True,
+                        temperature=0.01,  # 'randomness' of outputs, 0.0 is the min and 1.0 the max
+                        top_p=0.95,  # select from top tokens whose probability add up to 15%
+                        top_k=0,  # select from top 0 tokens (because zero, relies on top_p)
+                        max_new_tokens=max_new_tokens,  # mex number of tokens to generate in the output
+                        repetition_penalty=repetition_penalty,  # without this output begins repeating
+                    )
+                )
+                self.llm = HuggingFacePipeline(pipeline=pipe, callbacks=callbacks)
+            elif self.llm_model_type == "stablelm":
+                MODEL_NAME_OR_PATH = os.environ.get("STABLELM_MODEL_NAME_OR_PATH")
+                print(f"            loading model: {MODEL_NAME_OR_PATH}")
+
+                config = AutoConfig.from_pretrained(
+                    MODEL_NAME_OR_PATH, trust_remote_code=True
+                )
+                # config.attn_config["attn_impl"] = "triton"
+                # config.max_seq_len = 4096
+                config.init_device = hf_pipeline_device_type
+
+                SL_RP = os.environ.get("SL_RP")
+                if SL_RP is not None and len(SL_RP) > 0:
+                    repetition_penalty = float(SL_RP)
+                else:
+                    repetition_penalty = 1.05
+                print(f"       repetition_penalty: {repetition_penalty}")
+
+                model = (
+                    AutoModelForCausalLM.from_pretrained(
+                        MODEL_NAME_OR_PATH,
+                        config=config,
+                        quantization_config=double_quant_config,
+                        trust_remote_code=True,
+                    )
+                    if load_quantized_model is not None
+                    else AutoModelForCausalLM.from_pretrained(
+                        MODEL_NAME_OR_PATH,
+                        config=config,
+                        torch_dtype=torch_dtype,
+                        trust_remote_code=True,
+                    )
+                )
+
+                print(f"Model loaded on {config.init_device}")
+                print(f"Model memory footprint: {model.get_memory_footprint()}")
+
+                tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME_OR_PATH)
+                self._init_hf_streamer(tokenizer)
+
+                class StopOnTokens(StoppingCriteria):
+                    def __call__(
+                        self,
+                        input_ids: torch.LongTensor,
+                        scores: torch.FloatTensor,
+                        **kwargs,
+                    ) -> bool:
+                        stop_ids = [50278, 50279, 50277, 1, 0]
+                        for stop_id in stop_ids:
+                            if input_ids[0][-1] == stop_id:
+                                return True
+                        return False
+
+                stopping_criteria = StoppingCriteriaList([StopOnTokens()])
+
+                pipe = (
+                    pipeline(
+                        model=model,
+                        tokenizer=tokenizer,
+                        streamer=self.streamer,
+                        return_full_text=True,  # langchain expects the full text
+                        task="text-generation",
+                        device_map="auto",
+                        # we pass model parameters here too
+                        stopping_criteria=stopping_criteria,  # without this model will ramble
+                        temperature=0,  # 'randomness' of outputs, 0.0 is the min and 1.0 the max
+                        top_p=0.95,  # select from top tokens whose probability add up to 15%
+                        top_k=0,  # select from top 0 tokens (because zero, relies on top_p)
+                        max_new_tokens=2048,  # mex number of tokens to generate in the output
+                        repetition_penalty=repetition_penalty,  # without this output begins repeating
+                    )
+                    if load_quantized_model is not None
+                    else pipeline(
+                        model=model,
+                        tokenizer=tokenizer,
+                        streamer=self.streamer,
+                        return_full_text=True,  # langchain expects the full text
+                        task="text-generation",
+                        device=config.init_device,
+                        # we pass model parameters here too
+                        stopping_criteria=stopping_criteria,  # without this model will ramble
+                        temperature=0,  # 'randomness' of outputs, 0.0 is the min and 1.0 the max
+                        top_p=0.95,  # select from top tokens whose probability add up to 15%
+                        top_k=0,  # select from top 0 tokens (because zero, relies on top_p)
+                        max_new_tokens=2048,  # mex number of tokens to generate in the output
+                        repetition_penalty=repetition_penalty,  # without this output begins repeating
+                    )
+                )
+                self.llm = HuggingFacePipeline(pipeline=pipe, callbacks=callbacks)
+
+        print("initialization complete")

app_modules/llm_qa_chain.py

@@ -0,0 +1,20 @@
+from langchain.chains import ConversationalRetrievalChain
+from langchain.chains.base import Chain
+
+from app_modules.llm_inference import LLMInference
+
+
+class QAChain(LLMInference):
+    def __init__(self, vectorstore, llm_loader):
+        super().__init__(llm_loader)
+        self.vectorstore = vectorstore
+
+    def create_chain(self) -> Chain:
+        qa = ConversationalRetrievalChain.from_llm(
+            self.llm_loader.llm,
+            self.vectorstore.as_retriever(search_kwargs=self.llm_loader.search_kwargs),
+            max_tokens_limit=self.llm_loader.max_tokens_limit,
+            return_source_documents=True,
+        )
+
+        return qa

app_modules/utils.py

@@ -0,0 +1,166 @@
+# -*- coding:utf-8 -*-
+from __future__ import annotations
+
+import logging
+import os
+import platform
+import re
+from pathlib import Path
+
+import requests
+import torch
+from tqdm import tqdm
+
+
+class LogRecord(logging.LogRecord):
+    def getMessage(self):
+        msg = self.msg
+        if self.args:
+            if isinstance(self.args, dict):
+                msg = msg.format(**self.args)
+            else:
+                msg = msg.format(*self.args)
+        return msg
+
+
+class Logger(logging.Logger):
+    def makeRecord(
+        self,
+        name,
+        level,
+        fn,
+        lno,
+        msg,
+        args,
+        exc_info,
+        func=None,
+        extra=None,
+        sinfo=None,
+    ):
+        rv = LogRecord(name, level, fn, lno, msg, args, exc_info, func, sinfo)
+        if extra is not None:
+            for key in extra:
+                rv.__dict__[key] = extra[key]
+        return rv
+
+
+def init_settings():
+    logging.setLoggerClass(Logger)
+    logging.basicConfig(
+        level=logging.WARNING,
+        format="%(asctime)s [%(levelname)s] [%(filename)s:%(lineno)d] %(message)s",
+    )
+
+
+def remove_extra_spaces(text):
+    return re.sub(" +", " ", text.strip())
+
+
+def print_llm_response(llm_response):
+    answer = llm_response["answer"] if "answer" in llm_response else None
+    if answer is None:
+        answer = llm_response["token"] if "token" in llm_response else None
+
+    if answer is not None:
+        print("\n\n***Answer:")
+        print(answer)
+
+    source_documents = (
+        llm_response["source_documents"] if "source_documents" in llm_response else None
+    )
+    if source_documents is None:
+        source_documents = llm_response["sourceDocs"]
+
+    print("\nSources:")
+    for source in source_documents:
+        metadata = source["metadata"] if "metadata" in source else source.metadata
+        if "page" in metadata:
+            print(f"  Page:  {metadata['page']}", end="")
+
+        print(
+            " Source: "
+            + str(metadata["url"] if "url" in metadata else metadata["source"])
+        )
+        print(
+            source["page_content"] if "page_content" in source else source.page_content
+        )
+
+
+def get_device_types():
+    print("Running on: ", platform.platform())
+    print("MPS is", "NOT" if not torch.backends.mps.is_available() else "", "available")
+    print("CUDA is", "NOT" if not torch.cuda.is_available() else "", "available")
+    device_type_available = "cpu"
+
+    if not torch.backends.mps.is_available():
+        if not torch.backends.mps.is_built():
+            print(
+                "MPS not available because the current PyTorch install was not "
+                "built with MPS enabled."
+            )
+        else:
+            print(
+                "MPS not available because the current MacOS version is not 12.3+ "
+                "and/or you do not have an MPS-enabled device on this machine."
+            )
+    else:
+        device_type_available = "mps"
+
+    if torch.cuda.is_available():
+        print("CUDA is available, we have found ", torch.cuda.device_count(), " GPU(s)")
+        print(torch.cuda.get_device_name(0))
+        print("CUDA version: " + torch.version.cuda)
+        device_type_available = f"cuda:{torch.cuda.current_device()}"
+
+    return (
+        os.environ.get("HF_EMBEDDINGS_DEVICE_TYPE") or device_type_available,
+        os.environ.get("HF_PIPELINE_DEVICE_TYPE") or device_type_available,
+    )
+
+
+def ensure_model_is_downloaded(llm_model_type):
+    if llm_model_type.startswith("gpt4all"):
+        local_path = (
+            os.environ.get("GPT4ALL_J_MODEL_PATH")
+            if llm_model_type == "gpt4all-j"
+            else os.environ.get("GPT4ALL_MODEL_PATH")
+        )
+        url = (
+            os.environ.get("GPT4ALL_J_DOWNLOAD_LINK")
+            if llm_model_type == "gpt4all-j"
+            else os.environ.get("GPT4ALL_DOWNLOAD_LINK")
+        )
+    elif llm_model_type == "llamacpp":
+        local_path = os.environ.get("LLAMACPP_MODEL_PATH")
+        url = os.environ.get("LLAMACPP_DOWNLOAD_LINK")
+    elif llm_model_type == "ctransformers":
+        local_path = os.environ.get("CTRANSFORMERS_MODEL_PATH")
+        url = os.environ.get("CTRANSFORMERS_DOWNLOAD_LINK")
+    else:
+        raise ValueError(f"wrong model typle: {llm_model_type}")
+
+    path = Path(local_path)
+
+    if path.is_file():
+        print(f"model: {local_path} exists")
+    else:
+        print(f"downloading model: {local_path} from {url} ...")
+        path.parent.mkdir(parents=True, exist_ok=True)
+
+        # send a GET request to the URL to download the file. Stream since it's large
+        response = requests.get(url, stream=True)
+
+        # open the file in binary mode and write the contents of the response to it in chunks
+        # This is a large file, so be prepared to wait.
+        with open(local_path, "wb") as f:
+            for chunk in tqdm(response.iter_content(chunk_size=8192)):
+                if chunk:
+                    f.write(chunk)
+
+    return local_path
+
+
+if __name__ == "__main__":
+    hf_embeddings_device_type, hf_pipeline_device_type = get_device_types()
+    print(f"hf_embeddings_device_type: {hf_embeddings_device_type}")
+    print(f"hf_pipeline_device_type: {hf_pipeline_device_type}")

assets/custom.css

@@ -0,0 +1,488 @@
+:root {
+    --chatbot-color-light: #F3F3F3;
+    --chatbot-color-dark: #121111;
+}
+
+/* status_display */
+#status_display {
+    display: flex;
+    min-height: 2.5em;
+    align-items: flex-end;
+    justify-content: flex-end;
+}
+
+#status_display p {
+    font-size: .85em;
+    font-family: monospace;
+    color: var(--body-text-color-subdued);
+}
+
+
+
+/* usage_display */
+#usage_display {
+    height: 1em;
+}
+
+#usage_display p {
+    padding: 0 1em;
+    font-size: .85em;
+    font-family: monospace;
+    color: var(--body-text-color-subdued);
+}
+
+/* list */
+ol:not(.options),
+ul:not(.options) {
+    padding-inline-start: 2em !important;
+}
+
+/* Thank @Keldos-Li for fixing it */
+/* Light mode (default) */
+#inflaton_chatbot {
+    background-color: var(--chatbot-color-light) !important;
+    color: #000000 !important;
+}
+
+[data-testid="bot"] {
+    background-color: #FFFFFF !important;
+}
+
+[data-testid="user"] {
+    background-color: #95EC69 !important;
+}
+
+/* Dark mode */
+.dark #inflaton_chatbot {
+    background-color: var(--chatbot-color-dark) !important;
+    color: #FFFFFF !important;
+}
+
+.dark [data-testid="bot"] {
+    background-color: #2C2C2C !important;
+}
+
+.dark [data-testid="user"] {
+    background-color: #26B561 !important;
+}
+
+#inflaton_chatbot {
+    height: 100%;
+    min-height: 400px;
+}
+
+[class *="message"] {
+    border-radius: var(--radius-xl) !important;
+    border: none;
+    padding: var(--spacing-xl) !important;
+    font-size: var(--text-md) !important;
+    line-height: var(--line-md) !important;
+    min-height: calc(var(--text-md)*var(--line-md) + 2*var(--spacing-xl));
+    min-width: calc(var(--text-md)*var(--line-md) + 2*var(--spacing-xl));
+}
+
+[data-testid="bot"] {
+    max-width: 85%;
+    border-bottom-left-radius: 0 !important;
+}
+
+[data-testid="user"] {
+    max-width: 85%;
+    width: auto !important;
+    border-bottom-right-radius: 0 !important;
+}
+
+/* Table */
+table {
+    margin: 1em 0;
+    border-collapse: collapse;
+    empty-cells: show;
+}
+
+td,
+th {
+    border: 1.2px solid var(--border-color-primary) !important;
+    padding: 0.2em;
+}
+
+thead {
+    background-color: rgba(175, 184, 193, 0.2);
+}
+
+thead th {
+    padding: .5em .2em;
+}
+
+/* Inline code */
+#inflaton_chatbot code {
+    display: inline;
+    white-space: break-spaces;
+    border-radius: 6px;
+    margin: 0 2px 0 2px;
+    padding: .2em .4em .1em .4em;
+    background-color: rgba(175, 184, 193, 0.2);
+}
+
+/* Code block */
+#inflaton_chatbot pre code {
+    display: block;
+    overflow: auto;
+    white-space: pre;
+    background-color: hsla(0, 0%, 0%, 80%) !important;
+    border-radius: 10px;
+    padding: 1.4em 1.2em 0em 1.4em;
+    margin: 1.2em 2em 1.2em 0.5em;
+    color: #FFF;
+    box-shadow: 6px 6px 16px hsla(0, 0%, 0%, 0.2);
+}
+
+/* Hightlight */
+#inflaton_chatbot .highlight {
+    background-color: transparent
+}
+
+#inflaton_chatbot .highlight .hll {
+    background-color: #49483e
+}
+
+#inflaton_chatbot .highlight .c {
+    color: #75715e
+}
+
+/* Comment */
+#inflaton_chatbot .highlight .err {
+    color: #960050;
+    background-color: #1e0010
+}
+
+/* Error */
+#inflaton_chatbot .highlight .k {
+    color: #66d9ef
+}
+
+/* Keyword */
+#inflaton_chatbot .highlight .l {
+    color: #ae81ff
+}
+
+/* Literal */
+#inflaton_chatbot .highlight .n {
+    color: #f8f8f2
+}
+
+/* Name */
+#inflaton_chatbot .highlight .o {
+    color: #f92672
+}
+
+/* Operator */
+#inflaton_chatbot .highlight .p {
+    color: #f8f8f2
+}
+
+/* Punctuation */
+#inflaton_chatbot .highlight .ch {
+    color: #75715e
+}
+
+/* Comment.Hashbang */
+#inflaton_chatbot .highlight .cm {
+    color: #75715e
+}
+
+/* Comment.Multiline */
+#inflaton_chatbot .highlight .cp {
+    color: #75715e
+}
+
+/* Comment.Preproc */
+#inflaton_chatbot .highlight .cpf {
+    color: #75715e
+}
+
+/* Comment.PreprocFile */
+#inflaton_chatbot .highlight .c1 {
+    color: #75715e
+}
+
+/* Comment.Single */
+#inflaton_chatbot .highlight .cs {
+    color: #75715e
+}
+
+/* Comment.Special */
+#inflaton_chatbot .highlight .gd {
+    color: #f92672
+}
+
+/* Generic.Deleted */
+#inflaton_chatbot .highlight .ge {
+    font-style: italic
+}
+
+/* Generic.Emph */
+#inflaton_chatbot .highlight .gi {
+    color: #a6e22e
+}
+
+/* Generic.Inserted */
+#inflaton_chatbot .highlight .gs {
+    font-weight: bold
+}
+
+/* Generic.Strong */
+#inflaton_chatbot .highlight .gu {
+    color: #75715e
+}
+
+/* Generic.Subheading */
+#inflaton_chatbot .highlight .kc {
+    color: #66d9ef
+}
+
+/* Keyword.Constant */
+#inflaton_chatbot .highlight .kd {
+    color: #66d9ef
+}
+
+/* Keyword.Declaration */
+#inflaton_chatbot .highlight .kn {
+    color: #f92672
+}
+
+/* Keyword.Namespace */
+#inflaton_chatbot .highlight .kp {
+    color: #66d9ef
+}
+
+/* Keyword.Pseudo */
+#inflaton_chatbot .highlight .kr {
+    color: #66d9ef
+}
+
+/* Keyword.Reserved */
+#inflaton_chatbot .highlight .kt {
+    color: #66d9ef
+}
+
+/* Keyword.Type */
+#inflaton_chatbot .highlight .ld {
+    color: #e6db74
+}
+
+/* Literal.Date */
+#inflaton_chatbot .highlight .m {
+    color: #ae81ff
+}
+
+/* Literal.Number */
+#inflaton_chatbot .highlight .s {
+    color: #e6db74
+}
+
+/* Literal.String */
+#inflaton_chatbot .highlight .na {
+    color: #a6e22e
+}
+
+/* Name.Attribute */
+#inflaton_chatbot .highlight .nb {
+    color: #f8f8f2
+}
+
+/* Name.Builtin */
+#inflaton_chatbot .highlight .nc {
+    color: #a6e22e
+}
+
+/* Name.Class */
+#inflaton_chatbot .highlight .no {
+    color: #66d9ef
+}
+
+/* Name.Constant */
+#inflaton_chatbot .highlight .nd {
+    color: #a6e22e
+}
+
+/* Name.Decorator */
+#inflaton_chatbot .highlight .ni {
+    color: #f8f8f2
+}
+
+/* Name.Entity */
+#inflaton_chatbot .highlight .ne {
+    color: #a6e22e
+}
+
+/* Name.Exception */
+#inflaton_chatbot .highlight .nf {
+    color: #a6e22e
+}
+
+/* Name.Function */
+#inflaton_chatbot .highlight .nl {
+    color: #f8f8f2
+}
+
+/* Name.Label */
+#inflaton_chatbot .highlight .nn {
+    color: #f8f8f2
+}
+
+/* Name.Namespace */
+#inflaton_chatbot .highlight .nx {
+    color: #a6e22e
+}
+
+/* Name.Other */
+#inflaton_chatbot .highlight .py {
+    color: #f8f8f2
+}
+
+/* Name.Property */
+#inflaton_chatbot .highlight .nt {
+    color: #f92672
+}
+
+/* Name.Tag */
+#inflaton_chatbot .highlight .nv {
+    color: #f8f8f2
+}
+
+/* Name.Variable */
+#inflaton_chatbot .highlight .ow {
+    color: #f92672
+}
+
+/* Operator.Word */
+#inflaton_chatbot .highlight .w {
+    color: #f8f8f2
+}
+
+/* Text.Whitespace */
+#inflaton_chatbot .highlight .mb {
+    color: #ae81ff
+}
+
+/* Literal.Number.Bin */
+#inflaton_chatbot .highlight .mf {
+    color: #ae81ff
+}
+
+/* Literal.Number.Float */
+#inflaton_chatbot .highlight .mh {
+    color: #ae81ff
+}
+
+/* Literal.Number.Hex */
+#inflaton_chatbot .highlight .mi {
+    color: #ae81ff
+}
+
+/* Literal.Number.Integer */
+#inflaton_chatbot .highlight .mo {
+    color: #ae81ff
+}
+
+/* Literal.Number.Oct */
+#inflaton_chatbot .highlight .sa {
+    color: #e6db74
+}
+
+/* Literal.String.Affix */
+#inflaton_chatbot .highlight .sb {
+    color: #e6db74
+}
+
+/* Literal.String.Backtick */
+#inflaton_chatbot .highlight .sc {
+    color: #e6db74
+}
+
+/* Literal.String.Char */
+#inflaton_chatbot .highlight .dl {
+    color: #e6db74
+}
+
+/* Literal.String.Delimiter */
+#inflaton_chatbot .highlight .sd {
+    color: #e6db74
+}
+
+/* Literal.String.Doc */
+#inflaton_chatbot .highlight .s2 {
+    color: #e6db74
+}
+
+/* Literal.String.Double */
+#inflaton_chatbot .highlight .se {
+    color: #ae81ff
+}
+
+/* Literal.String.Escape */
+#inflaton_chatbot .highlight .sh {
+    color: #e6db74
+}
+
+/* Literal.String.Heredoc */
+#inflaton_chatbot .highlight .si {
+    color: #e6db74
+}
+
+/* Literal.String.Interpol */
+#inflaton_chatbot .highlight .sx {
+    color: #e6db74
+}
+
+/* Literal.String.Other */
+#inflaton_chatbot .highlight .sr {
+    color: #e6db74
+}
+
+/* Literal.String.Regex */
+#inflaton_chatbot .highlight .s1 {
+    color: #e6db74
+}
+
+/* Literal.String.Single */
+#inflaton_chatbot .highlight .ss {
+    color: #e6db74
+}
+
+/* Literal.String.Symbol */
+#inflaton_chatbot .highlight .bp {
+    color: #f8f8f2
+}
+
+/* Name.Builtin.Pseudo */
+#inflaton_chatbot .highlight .fm {
+    color: #a6e22e
+}
+
+/* Name.Function.Magic */
+#inflaton_chatbot .highlight .vc {
+    color: #f8f8f2
+}
+
+/* Name.Variable.Class */
+#inflaton_chatbot .highlight .vg {
+    color: #f8f8f2
+}
+
+/* Name.Variable.Global */
+#inflaton_chatbot .highlight .vi {
+    color: #f8f8f2
+}
+
+/* Name.Variable.Instance */
+#inflaton_chatbot .highlight .vm {
+    color: #f8f8f2
+}
+
+/* Name.Variable.Magic */
+#inflaton_chatbot .highlight .il {
+    color: #ae81ff
+}
+
+/* Literal.Number.Integer.Long */

data/faiss_1024_512/index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98e8c49e6c3ef2bcd0b258fb51ffe58fa92a63544b672f1c0c75857593afa2a8
+size 5987373

data/faiss_1024_512/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8467d3647bf272f11151a512f54515ef6dd83f2081686156a437132380b28b4b
+size 2035755

data/questions.txt

@@ -0,0 +1,4 @@
+What's PCI DSS?
+Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?
+new requirements for vulnerability assessments
+more on penetration testing

eval_models.py

@@ -0,0 +1,230 @@
+import ast
+import codecs
+import json
+from ragas import evaluate
+from ragas.metrics import answer_relevancy, faithfulness
+from datasets import Dataset
+from langchain_openai.chat_models import ChatOpenAI
+
+
+from app_modules.init import app_init
+
+llm_loader, qa_chain = app_init()
+qa = qa_chain.get_chain()
+
+gpt4_turbo = ChatOpenAI(model_name="gpt-4-turbo-preview")
+
+
+def load_notebook(filename, print_source=False):
+    f = codecs.open(filename, "r")
+    source = f.read()
+
+    print("loading: ", filename)
+    notebook = json.loads(source)
+
+    if print_source:
+        pySource = f"### Python code from {filename}:\n"
+        for x in notebook["cells"]:
+            for x2 in x["source"]:
+                pySource = pySource + x2
+                if x2[-1] != "\n":
+                    pySource = pySource + "\n"
+
+        print(pySource)
+    return notebook
+
+
+def parse_outputs(outputs):
+    questions = [
+        "Question: What's PCI DSS?\n",
+        "Question: Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?\n",
+        "Question: new requirements for vulnerability assessments\n",
+        "Question: more on penetration testing\n",
+    ]
+    result = []
+    for question in questions:
+        start = outputs.index(question)
+        conversation = ast.literal_eval(outputs[start + 1])
+        index = start + 2
+
+        if len(conversation["chat_history"]) > 0:
+            conversation["standalone_question"] = ""
+            while not outputs[index].startswith("<on_llm_end>"):
+                conversation["standalone_question"] += outputs[index]
+                index += 1
+
+            index += 1
+            while outputs[index] == "\n":
+                index += 1
+
+        conversation["answer"] = ""
+
+        while not outputs[index].startswith("<on_llm_end>"):
+            conversation["answer"] += outputs[index]
+            index += 1
+
+        while not outputs[index].startswith("Completed"):
+            index += 1
+
+        timing = outputs[index].split()[-1]
+        conversation["time_used_in_seconds"] = timing[:-1]
+
+        result.append(conversation)
+
+    index += 1
+    total_time_used = outputs[index].split()[-2]
+
+    index += 1
+    num_tokens_generated = outputs[index].split()[-1]
+
+    index += 1
+    token_per_second = outputs[index].split()[-2]
+
+    return {
+        "conversations": result,
+        "total_time_used": total_time_used,
+        "num_tokens_generated": num_tokens_generated,
+        "token_per_second": token_per_second,
+    }
+
+
+def parse_results(notebook):
+    result = {}
+    repetition_penalty = None
+    for x in notebook["cells"]:
+        source = x["source"]
+        for x2 in source:
+            # print(x2)
+            if "_RP" in x2:
+                start = x2.index("1.")
+                end = x2.index('"', start)
+                repetition_penalty = x2[start:end]
+                print("processing repetition_penalty:", repetition_penalty)
+
+        if source and repetition_penalty:
+            outputs = x["outputs"][0]["text"]
+            result[repetition_penalty] = parse_outputs(outputs)
+            repetition_penalty = None
+
+    return result
+
+
+def calc_ragas_scores(conversations):
+    dict = {
+        "question": [],
+        "user_question": [],
+        "standalone_question": [],
+        "contexts": [],
+        "answer": [],
+    }
+
+    for conversation in conversations:
+        standalone_question = (
+            conversation["standalone_question"]
+            if "standalone_question" in conversation
+            else conversation["question"]
+        )
+        dict["question"].append(standalone_question)
+        dict["answer"].append(conversation["answer"])
+
+        dict["user_question"].append(conversation["question"])
+        dict["standalone_question"].append(
+            conversation["standalone_question"]
+            if "standalone_question" in conversation
+            else ""
+        )
+
+        contexts = []
+        docs = qa.retriever.get_relevant_documents(standalone_question)
+        for doc in docs:
+            contexts.append(doc.page_content)
+
+        dict["contexts"].append(contexts)
+
+    # print(dict)
+
+    ds = Dataset.from_dict(dict)
+
+    result = evaluate(
+        ds,
+        metrics=[
+            faithfulness,
+            answer_relevancy,
+        ],
+        llm=gpt4_turbo,
+    )
+
+    result["overall_score"] = 2 / (
+        1 / result["faithfulness"] + 1 / result["answer_relevancy"]
+    )
+
+    print(f"\n\n# Ragas scores: {result}\n")
+    return dict, result
+
+
+def evaluate_models(model_names, prefix="nvidia-4090"):
+    raw_data = {
+        "model_name": [],
+        "repetition_penalty": [],
+        "user_question": [],
+        "standalone_question": [],
+        "contexts": [],
+        "answer": [],
+    }
+    perf_data = {
+        "model_name": [],
+        "repetition_penalty": [],
+        "faithfulness": [],
+        "answer_relevancy": [],
+        "overall_score": [],
+        "total_time_used": [],
+        "num_tokens_generated": [],
+        "token_per_second": [],
+    }
+
+    repetition_penalties = ["1.05", "1.10", "1.15"]
+
+    openai_model_names = {
+        "1.05": "gpt-3.5-turbo",
+        "1.10": "gpt-3.5-turbo-instruct",
+        "1.15": "gpt-4",
+    }
+
+    for model_name in model_names:
+        notebook = load_notebook(f"./notebook/{prefix}-{model_name}.ipynb")
+        results = parse_results(notebook)
+        for repetition_penalty in repetition_penalties:
+            result = results[repetition_penalty]
+            dict, ragas = calc_ragas_scores(result["conversations"])
+
+            if model_name == "openai" or model_name.startswith("gpt-"):
+                model_name = openai_model_names[repetition_penalty]
+                repetition_penalty = ""
+
+            for _ in dict["question"]:
+                raw_data["model_name"].append(model_name)
+                raw_data["repetition_penalty"].append(repetition_penalty)
+
+            raw_data["user_question"] += dict["user_question"]
+            raw_data["standalone_question"] += dict["standalone_question"]
+            raw_data["contexts"] += dict["contexts"]
+            raw_data["answer"] += dict["answer"]
+
+            perf_data["model_name"].append(model_name)
+            perf_data["repetition_penalty"].append(repetition_penalty)
+
+            perf_data["faithfulness"].append(ragas["faithfulness"])
+            perf_data["answer_relevancy"].append(ragas["answer_relevancy"])
+            perf_data["overall_score"].append(ragas["overall_score"])
+            perf_data["num_tokens_generated"].append(
+                int(result["num_tokens_generated"])
+            )
+            perf_data["total_time_used"].append(float(result["total_time_used"]))
+            perf_data["token_per_second"].append(float(result["token_per_second"]))
+
+    perf_ds = Dataset.from_dict(perf_data)
+    perf_pd = perf_ds.to_pandas()
+
+    raw_ds = Dataset.from_dict(raw_data)
+    raw_pd = raw_ds.to_pandas()
+    return perf_pd, raw_pd

ingest.py

@@ -0,0 +1,129 @@
+# setting device on GPU if available, else CPU
+import os
+from timeit import default_timer as timer
+from typing import List
+
+from langchain.document_loaders import PyPDFDirectoryLoader
+from langchain_community.embeddings import HuggingFaceInstructEmbeddings
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.vectorstores.base import VectorStore
+from langchain.vectorstores.chroma import Chroma
+from langchain.vectorstores.faiss import FAISS
+
+from app_modules.init import *
+
+
+def load_documents(source_pdfs_path, urls) -> List:
+    loader = PyPDFDirectoryLoader(source_pdfs_path, silent_errors=True)
+    documents = loader.load()
+    if urls is not None and len(urls) > 0:
+        for doc in documents:
+            source = doc.metadata["source"]
+            filename = source.split("/")[-1]
+            for url in urls:
+                if url.endswith(filename):
+                    doc.metadata["url"] = url
+                    break
+    return documents
+
+
+def split_chunks(documents: List, chunk_size, chunk_overlap) -> List:
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=chunk_size, chunk_overlap=chunk_overlap
+    )
+    return text_splitter.split_documents(documents)
+
+
+def generate_index(
+    chunks: List, embeddings: HuggingFaceInstructEmbeddings
+) -> VectorStore:
+    if using_faiss:
+        faiss_instructor_embeddings = FAISS.from_documents(
+            documents=chunks, embedding=embeddings
+        )
+
+        faiss_instructor_embeddings.save_local(index_path)
+        return faiss_instructor_embeddings
+    else:
+        chromadb_instructor_embeddings = Chroma.from_documents(
+            documents=chunks, embedding=embeddings, persist_directory=index_path
+        )
+
+        chromadb_instructor_embeddings.persist()
+        return chromadb_instructor_embeddings
+
+
+# Constants
+device_type, hf_pipeline_device_type = get_device_types()
+hf_embeddings_model_name = (
+    os.environ.get("HF_EMBEDDINGS_MODEL_NAME") or "hkunlp/instructor-xl"
+)
+index_path = os.environ.get("FAISS_INDEX_PATH") or os.environ.get("CHROMADB_INDEX_PATH")
+using_faiss = os.environ.get("FAISS_INDEX_PATH") is not None
+source_pdfs_path = os.environ.get("SOURCE_PDFS_PATH")
+source_urls = os.environ.get("SOURCE_URLS")
+chunk_size = os.environ.get("CHUNCK_SIZE")
+chunk_overlap = os.environ.get("CHUNK_OVERLAP")
+
+start = timer()
+embeddings = HuggingFaceInstructEmbeddings(
+    model_name=hf_embeddings_model_name, model_kwargs={"device": device_type}
+)
+end = timer()
+
+print(f"Completed in {end - start:.3f}s")
+
+start = timer()
+
+if not os.path.isdir(index_path):
+    print(
+        f"The index persist directory {index_path} is not present. Creating a new one."
+    )
+    os.mkdir(index_path)
+
+    if source_urls is not None:
+        # Open the file for reading
+        file = open(source_urls, "r")
+
+        # Read the contents of the file into a list of strings
+        lines = file.readlines()
+
+        # Close the file
+        file.close()
+
+        # Remove the newline characters from each string
+        source_urls = [line.strip() for line in lines]
+
+    print(
+        f"Loading {'' if source_urls is None else str(len(source_urls)) + ' '}PDF files from {source_pdfs_path}"
+    )
+    sources = load_documents(source_pdfs_path, source_urls)
+
+    print(f"Splitting {len(sources)} PDF pages in to chunks ...")
+
+    chunks = split_chunks(
+        sources, chunk_size=int(chunk_size), chunk_overlap=int(chunk_overlap)
+    )
+    print(f"Generating index for {len(chunks)} chunks ...")
+
+    index = generate_index(chunks, embeddings)
+else:
+    print(f"The index persist directory {index_path} is present. Loading index ...")
+    index = (
+        FAISS.load_local(index_path, embeddings)
+        if using_faiss
+        else Chroma(embedding_function=embeddings, persist_directory=index_path)
+    )
+    query = "hi"
+    print(f"Load relevant documents for standalone question: {query}")
+
+    start2 = timer()
+    docs = index.as_retriever().get_relevant_documents(query)
+    end = timer()
+
+    print(f"Completed in {end - start2:.3f}s")
+    print(docs)
+
+end = timer()
+
+print(f"Completed in {end - start:.3f}s")

notebook/nvidia-4080-llama-2-7b.ipynb

@@ -0,0 +1,2459 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "874cd58e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "workding dir: /home/inflaton/code/projects/chat-with-orca-2\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import sys\n",
+    "from pathlib import Path\n",
+    "\n",
+    "workding_dir = str(Path.cwd().parent)\n",
+    "os.chdir(workding_dir)\n",
+    "sys.path.append(workding_dir)\n",
+    "print(\"workding dir:\", workding_dir)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "19f967ca",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "os.environ[\"LLM_MODEL_TYPE\"] = \"huggingface\"\n",
+    "os.environ[\"HUGGINGFACE_MODEL_NAME_OR_PATH\"] = \"meta-llama/Llama-2-7b-chat-hf\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "a6d96660",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading env vars from: /home/inflaton/code/projects/chat-with-orca-2/.env\n",
+      "Running on:  Linux-5.10.102.1-microsoft-standard-WSL2-x86_64-with-glibc2.35\n",
+      "MPS is NOT available\n",
+      "CUDA is  available\n",
+      "MPS not available because the current PyTorch install was not built with MPS enabled.\n",
+      "CUDA is available, we have found  1  GPU(s)\n",
+      "NVIDIA GeForce RTX 4080 Laptop GPU\n",
+      "CUDA version: 12.1\n",
+      "hf_embeddings_device_type: cuda:0\n",
+      "hf_pipeline_device_type: cuda:0\n",
+      "load INSTRUCTOR_Transformer\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n",
+      "max_seq_length  512\n",
+      "Completed in 0.610s\n",
+      "Load index from ./data/faiss_1024_512/ with FAISS\n",
+      "Completed in 0.019s\n",
+      "initializing LLM: huggingface\n",
+      "  hf_pipeline_device_type: cuda:0\n",
+      "     load_quantized_model: None\n",
+      "              torch_dtype: torch.float16\n",
+      "                 n_threds: 4\n",
+      "            loading model: meta-llama/Llama-2-7b-chat-hf\n",
+      "            HF auth token: None\n",
+      "       repetition_penalty: 1.05\n",
+      "Loading checkpoint shards: 100%|██████████████████| 2/2 [02:47<00:00, 83.80s/it]\n",
+      "Model memory footprint: 13543948288\n",
+      "initialization complete\n",
+      "Completed in 172.421s\n",
+      "\n",
+      "Question: What's PCI DSS?\n",
+      "{'question': \"What's PCI DSS?\", 'chat_history': []}\n",
+      "PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment account data. The standard is developed and maintained by the PCI Security Standards Council, and it includes requirements for building and maintaining a secure network and systems, protecting account data, maintaining a vulnerability management program, implementing strong access control measures, and more. The goal of PCI DSS is to protect payment data from unauthorized access, use, disclosure, disruption, modification, or destruction.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 62.095s\n",
+      "\n",
+      "Question: Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?\n",
+      "{'question': 'Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment account data. The standard is developed and maintained by the PCI Security Standards Council, and it includes requirements for building and maintaining a secure network and systems, protecting account data, maintaining a vulnerability management program, implementing strong access control measures, and more. The goal of PCI DSS is to protect payment data from unauthorized access, use, disclosure, disruption, modification, or destruction.')]}\n",
+      "What are the key changes made in PCI DSS version 4.0 compared to version 3.2.1?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " The key changes made in PCI DSS version 4.0 compared to version 3.2.1 include updated principal requirement titles, replaced \"firewalls\" and \"routers\" with \"network security controls\", and evolved requirements related to roles and responsibilities. Additionally, there are errata updates to correct entries in the Summary of New Requirements table and to add descriptions of changes made to certain requirements.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 56.760s\n",
+      "\n",
+      "Question: new requirements for vulnerability assessments\n",
+      "{'question': 'new requirements for vulnerability assessments', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment account data. The standard is developed and maintained by the PCI Security Standards Council, and it includes requirements for building and maintaining a secure network and systems, protecting account data, maintaining a vulnerability management program, implementing strong access control measures, and more. The goal of PCI DSS is to protect payment data from unauthorized access, use, disclosure, disruption, modification, or destruction.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'The key changes made in PCI DSS version 4.0 compared to version 3.2.1 include updated principal requirement titles, replaced \"firewalls\" and \"routers\" with \"network security controls\", and evolved requirements related to roles and responsibilities. Additionally, there are errata updates to correct entries in the Summary of New Requirements table and to add descriptions of changes made to certain requirements.')]}\n",
+      "What are the updated requirements for vulnerability assessments in PCI DSS version 4.0 compared to version 3.2.1?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " The updated requirements for vulnerability assessments in PCI DSS version 4.0 are as follows:\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 1051.367s\n",
+      "\n",
+      "Question: more on penetration testing\n",
+      "{'question': 'more on penetration testing', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment account data. The standard is developed and maintained by the PCI Security Standards Council, and it includes requirements for building and maintaining a secure network and systems, protecting account data, maintaining a vulnerability management program, implementing strong access control measures, and more. The goal of PCI DSS is to protect payment data from unauthorized access, use, disclosure, disruption, modification, or destruction.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'The key changes made in PCI DSS version 4.0 compared to version 3.2.1 include updated principal requirement titles, replaced \"firewalls\" and \"routers\" with \"network security controls\", and evolved requirements related to roles and responsibilities. Additionally, there are errata updates to correct entries in the Summary of New Requirements table and to add descriptions of changes made to certain requirements.'), ('new requirements for vulnerability assessments', 'The updated requirements for vulnerability assessments in PCI DSS version 4.0 are as follows:')]}\n",
+      "What are the updated requirements for vulnerability assessments in PCI DSS version 4.0?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " The updated requirements for vulnerability assessments in PCI DSS version 4.0 can be found in Section 11 of the summary of changes document. According to the document, there are new requirements for managing all other applicable vulnerabilities (those not ranked as high-risk or critical) found during internal vulnerability scans, and for performing vulnerability assessments via authenticated scanning. Additionally, there is a new requirement for maintaining an inventory of trusted keys and certificates.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 64.196s\n",
+      "Total time used: 1234.418 s\n",
+      "Number of tokens generated: 2457\n",
+      "Average generation speed: 1.990 tokens/s\n",
+      "CPU times: user 18.5 s, sys: 5.39 s, total: 23.8 s\n",
+      "Wall time: 23min 32s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "os.environ[\"HF_RP\"] = \"1.05\"\n",
+    "!python qa_chain_test.py"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "8a022a7b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading env vars from: /home/inflaton/code/projects/chat-with-orca-2/.env\n",
+      "Running on:  Linux-5.10.102.1-microsoft-standard-WSL2-x86_64-with-glibc2.35\n",
+      "MPS is NOT available\n",
+      "CUDA is  available\n",
+      "MPS not available because the current PyTorch install was not built with MPS enabled.\n",
+      "CUDA is available, we have found  1  GPU(s)\n",
+      "NVIDIA GeForce RTX 4080 Laptop GPU\n",
+      "CUDA version: 12.1\n",
+      "hf_embeddings_device_type: cuda:0\n",
+      "hf_pipeline_device_type: cuda:0\n",
+      "load INSTRUCTOR_Transformer\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n",
+      "max_seq_length  512\n",
+      "Completed in 0.647s\n",
+      "Load index from ./data/faiss_1024_512/ with FAISS\n",
+      "Completed in 0.021s\n",
+      "initializing LLM: huggingface\n",
+      "  hf_pipeline_device_type: cuda:0\n",
+      "     load_quantized_model: None\n",
+      "              torch_dtype: torch.float16\n",
+      "                 n_threds: 4\n",
+      "            loading model: meta-llama/Llama-2-7b-chat-hf\n",
+      "            HF auth token: None\n",
+      "       repetition_penalty: 1.1\n",
+      "Loading checkpoint shards: 100%|█████████████████| 2/2 [03:45<00:00, 112.81s/it]\n",
+      "Model memory footprint: 13543948288\n",
+      "initialization complete\n",
+      "Completed in 260.223s\n",
+      "\n",
+      "Question: What's PCI DSS?\n",
+      "{'question': \"What's PCI DSS?\", 'chat_history': []}\n",
+      "PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment account data. The standard is developed and maintained by the PCI Security Standards Council, and it includes requirements for building and maintaining a secure network and systems, protecting account data, maintaining a vulnerability management program, implementing strong access control measures, and more.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 62.521s\n",
+      "\n",
+      "Question: Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?\n",
+      "{'question': 'Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment account data. The standard is developed and maintained by the PCI Security Standards Council, and it includes requirements for building and maintaining a secure network and systems, protecting account data, maintaining a vulnerability management program, implementing strong access control measures, and more.')]}\n",
+      "What are the key changes made in PCI DSS version 4.0 compared to version 3.2.1?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " The key changes made in PCI DSS version 4.0 compared to version 3.2.1 include updated principal requirement titles, replaced \"firewalls\" and \"routers\" with \"network security controls\", and evolved requirements related to roles and responsibilities. Additionally, there were errata updates made in May 2022 and December 2022 to correct errors and provide additional information. However, I cannot provide a detailed answer without knowing your specific context or industry.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 50.686s\n",
+      "\n",
+      "Question: new requirements for vulnerability assessments\n",
+      "{'question': 'new requirements for vulnerability assessments', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment account data. The standard is developed and maintained by the PCI Security Standards Council, and it includes requirements for building and maintaining a secure network and systems, protecting account data, maintaining a vulnerability management program, implementing strong access control measures, and more.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'The key changes made in PCI DSS version 4.0 compared to version 3.2.1 include updated principal requirement titles, replaced \"firewalls\" and \"routers\" with \"network security controls\", and evolved requirements related to roles and responsibilities. Additionally, there were errata updates made in May 2022 and December 2022 to correct errors and provide additional information. However, I cannot provide a detailed answer without knowing your specific context or industry.')]}\n",
+      "What are the new requirements for vulnerability assessments in PCI DSS version 4.0?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " According to the provided summary of changes, there are no new requirements for vulnerability assessments in PCI DSS version 4.0. The existing requirements for vulnerability assessments remain unchanged from version 3.2.1 to version 4.0.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 42.544s\n",
+      "\n",
+      "Question: more on penetration testing\n",
+      "{'question': 'more on penetration testing', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment account data. The standard is developed and maintained by the PCI Security Standards Council, and it includes requirements for building and maintaining a secure network and systems, protecting account data, maintaining a vulnerability management program, implementing strong access control measures, and more.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'The key changes made in PCI DSS version 4.0 compared to version 3.2.1 include updated principal requirement titles, replaced \"firewalls\" and \"routers\" with \"network security controls\", and evolved requirements related to roles and responsibilities. Additionally, there were errata updates made in May 2022 and December 2022 to correct errors and provide additional information. However, I cannot provide a detailed answer without knowing your specific context or industry.'), ('new requirements for vulnerability assessments', 'According to the provided summary of changes, there are no new requirements for vulnerability assessments in PCI DSS version 4.0. The existing requirements for vulnerability assessments remain unchanged from version 3.2.1 to version 4.0.')]}\n",
+      "What are the key differences between penetration testing and vulnerability assessments in the context of PCI DSS compliance?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Penetration testing and vulnerability assessments are both important components of PCI DSS compliance, but they serve different purposes and have distinct methodologies.\n",
+      "Penetration testing involves simulating a cyber attack on an organization's network, systems, and applications to identify vulnerabilities that could be exploited by attackers. This type of testing is focused on identifying potential threats and weaknesses in the system, rather than simply identifying vulnerabilities. The goal of penetration testing is to provide recommendations for improving security posture and reducing the likelihood of a successful attack.\n",
+      "Vulnerability assessments, on the other hand, involve identifying and classifying potential vulnerabilities in the system based on industry standards and best practices. This type of assessment is focused on identifying potential weaknesses in the system and prioritizing remediation efforts based on risk. Vulnerability assessments typically use automated tools to scan the system for known vulnerabilities and may also include manual testing to identify additional vulnerabilities.\n",
+      "In summary, penetration testing is a more comprehensive and proactive approach to identifying security threats, while vulnerability assessments are a more reactive approach to identifying potential vulnerabilities. Both are important components of PCI DSS compliance and should be conducted regularly to ensure the security of cardholder data.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 116.722s\n",
+      "Total time used: 272.474 s\n",
+      "Number of tokens generated: 623\n",
+      "Average generation speed: 2.286 tokens/s\n",
+      "CPU times: user 6.52 s, sys: 1.93 s, total: 8.45 s\n",
+      "Wall time: 8min 57s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "os.environ[\"HF_RP\"] = \"1.10\"\n",
+    "!python qa_chain_test.py"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "db48dcc4",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading env vars from: /home/inflaton/code/projects/chat-with-orca-2/.env\n",
+      "Running on:  Linux-5.10.102.1-microsoft-standard-WSL2-x86_64-with-glibc2.35\n",
+      "MPS is NOT available\n",
+      "CUDA is  available\n",
+      "MPS not available because the current PyTorch install was not built with MPS enabled.\n",
+      "CUDA is available, we have found  1  GPU(s)\n",
+      "NVIDIA GeForce RTX 4080 Laptop GPU\n",
+      "CUDA version: 12.1\n",
+      "hf_embeddings_device_type: cuda:0\n",
+      "hf_pipeline_device_type: cuda:0\n",
+      "load INSTRUCTOR_Transformer\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n",
+      "max_seq_length  512\n",
+      "Completed in 0.611s\n",
+      "Load index from ./data/faiss_1024_512/ with FAISS\n",
+      "Completed in 0.027s\n",
+      "initializing LLM: huggingface\n",
+      "  hf_pipeline_device_type: cuda:0\n",
+      "     load_quantized_model: None\n",
+      "              torch_dtype: torch.float16\n",
+      "                 n_threds: 4\n",
+      "            loading model: meta-llama/Llama-2-7b-chat-hf\n",
+      "            HF auth token: None\n",
+      "       repetition_penalty: 1.15\n",
+      "Loading checkpoint shards: 100%|█████████████████| 2/2 [03:40<00:00, 110.30s/it]\n",
+      "Model memory footprint: 13543948288\n",
+      "initialization complete\n",
+      "Completed in 254.385s\n",
+      "\n",
+      "Question: What's PCI DSS?\n",
+      "{'question': \"What's PCI DSS?\", 'chat_history': []}\n",
+      "PCI DSS stands for Payment Card Industry Data Security Standard. It is a set of security standards designed to ensure that companies that handle credit card transactions keep customer data safe from cyber threats. These standards are developed and maintained by the PCI Security Standards Council, a global forum that brings together industry leaders to address security concerns. By adhering to these standards, companies can help prevent data breaches and protect their customers' sensitive information.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 63.409s\n",
+      "\n",
+      "Question: Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?\n",
+      "{'question': 'Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'chat_history': [(\"What's PCI DSS?\", \"PCI DSS stands for Payment Card Industry Data Security Standard. It is a set of security standards designed to ensure that companies that handle credit card transactions keep customer data safe from cyber threats. These standards are developed and maintained by the PCI Security Standards Council, a global forum that brings together industry leaders to address security concerns. By adhering to these standards, companies can help prevent data breaches and protect their customers' sensitive information.\")]}\n",
+      "What are the key changes between PCI DSS versions 3.2.1 and 4.0?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "The key changes between PCI DSS versions 3.2.1 and 4.0 include updates to align with changes from PCI DSS v3.2 to PCI DSS v4.0, minor corrections and edits made for clarification and/or format, and additional changes per requirement. However, if you don't know the answer, it's okay to say so instead of trying to make something up.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 49.842s\n",
+      "\n",
+      "Question: new requirements for vulnerability assessments\n",
+      "{'question': 'new requirements for vulnerability assessments', 'chat_history': [(\"What's PCI DSS?\", \"PCI DSS stands for Payment Card Industry Data Security Standard. It is a set of security standards designed to ensure that companies that handle credit card transactions keep customer data safe from cyber threats. These standards are developed and maintained by the PCI Security Standards Council, a global forum that brings together industry leaders to address security concerns. By adhering to these standards, companies can help prevent data breaches and protect their customers' sensitive information.\"), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', \"The key changes between PCI DSS versions 3.2.1 and 4.0 include updates to align with changes from PCI DSS v3.2 to PCI DSS v4.0, minor corrections and edits made for clarification and/or format, and additional changes per requirement. However, if you don't know the answer, it's okay to say so instead of trying to make something up.\")]}\n",
+      "What are some of the new requirements for vulnerability assessments in PCI DSS version 4.0 compared to previous versions?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "In PCI DSS version 4.0, there are several new requirements related to vulnerability assessments, including:\n",
+      "\n",
+      "* A new requirement (4.1.2) for roles and responsibilities related to vulnerability management.\n",
+      "* An updated bullet point (4.2.1) regarding the use of certificates for PAN transmissions over open, public networks, which includes a new best practice for maintaining an inventory of trusted keys and certificates.\n",
+      "* A new requirement (4.2.1.1) to ensure that only authorized personnel have access to trusted keys and certificates.\n",
+      "These changes aim to enhance the security posture of organizations handling cardholder data by providing more detailed guidelines for identifying and managing potential vulnerabilities.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 77.258s\n",
+      "\n",
+      "Question: more on penetration testing\n",
+      "{'question': 'more on penetration testing', 'chat_history': [(\"What's PCI DSS?\", \"PCI DSS stands for Payment Card Industry Data Security Standard. It is a set of security standards designed to ensure that companies that handle credit card transactions keep customer data safe from cyber threats. These standards are developed and maintained by the PCI Security Standards Council, a global forum that brings together industry leaders to address security concerns. By adhering to these standards, companies can help prevent data breaches and protect their customers' sensitive information.\"), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', \"The key changes between PCI DSS versions 3.2.1 and 4.0 include updates to align with changes from PCI DSS v3.2 to PCI DSS v4.0, minor corrections and edits made for clarification and/or format, and additional changes per requirement. However, if you don't know the answer, it's okay to say so instead of trying to make something up.\"), ('new requirements for vulnerability assessments', 'In PCI DSS version 4.0, there are several new requirements related to vulnerability assessments, including:\\n\\n* A new requirement (4.1.2) for roles and responsibilities related to vulnerability management.\\n* An updated bullet point (4.2.1) regarding the use of certificates for PAN transmissions over open, public networks, which includes a new best practice for maintaining an inventory of trusted keys and certificates.\\n* A new requirement (4.2.1.1) to ensure that only authorized personnel have access to trusted keys and certificates.\\nThese changes aim to enhance the security posture of organizations handling cardholder data by providing more detailed guidelines for identifying and managing potential vulnerabilities.')]}\n",
+      "What are some common techniques used in penetration testing to identify vulnerabilities in a system or network?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Penetration testing involves various techniques to identify vulnerabilities in a system or network. Some common techniques include:\n",
+      "\n",
+      "• Vulnerability scanning: This technique involves identifying potential vulnerabilities in a system or network using automated tools. The tool scans the network or system and reports any identified vulnerabilities.\n",
+      "• Social engineering: This technique involves manipulating individuals into divulging sensitive information or gaining unauthorized access to a system or network. Social engineers may use various tactics such as phishing emails, phone calls, or physical interactions to obtain information.\n",
+      "• Network traffic analysis: This technique involves monitoring and analyzing network traffic to identify suspicious activity or potential vulnerabilities. Tools such as Wireshark or Tcpdump can be used to capture and analyze network packets.\n",
+      "• Web application scanning: This technique involves identifying vulnerabilities in web applications using automated tools. The tool scans the application and reports any identified vulnerabilities.\n",
+      "• Malware analysis: This technique involves analyzing malware samples to identify their capabilities and potential vulnerabilities. This can help identify potential attack vectors and inform mitigation strategies.\n",
+      "• Compliance checking: This technique involves verifying that a system or network complies with relevant industry standards or regulations. By identifying non-compliant areas, organizations can prioritize remediation efforts.\n",
+      "• Penetration testing frameworks: These frameworks provide structured approaches to conducting penetration tests. They typically involve creating a detailed test plan, executing the test, and reporting findings. Common penetration testing frameworks include OWASP ZAP, NIST SP 800-115, and OSSTMM.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 129.022s\n",
+      "Total time used: 319.531 s\n",
+      "Number of tokens generated: 792\n",
+      "Average generation speed: 2.479 tokens/s\n",
+      "CPU times: user 7.12 s, sys: 2.03 s, total: 9.16 s\n",
+      "Wall time: 9min 38s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "os.environ[\"HF_RP\"] = \"1.15\"\n",
+    "!python qa_chain_test.py"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebook/nvidia-4080-orca-2-13b_tgi.ipynb

@@ -0,0 +1,359 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "874cd58e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "workding dir: /home/inflaton/code/projects/chat-with-orca-2\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import sys\n",
+    "from pathlib import Path\n",
+    "\n",
+    "workding_dir = str(Path.cwd().parent)\n",
+    "os.chdir(workding_dir)\n",
+    "sys.path.append(workding_dir)\n",
+    "print(\"workding dir:\", workding_dir)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "19f967ca",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "os.environ[\"LLM_MODEL_TYPE\"] = \"hftgi\"\n",
+    "os.environ[\"HFTGI_SERVER_URL\"] = \"https://enabled-factually-cougar.ngrok-free.app\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "a6d96660",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading env vars from: /home/inflaton/code/projects/chat-with-orca-2/.env\n",
+      "Running on:  Linux-5.10.102.1-microsoft-standard-WSL2-x86_64-with-glibc2.35\n",
+      "MPS is NOT available\n",
+      "CUDA is  available\n",
+      "MPS not available because the current PyTorch install was not built with MPS enabled.\n",
+      "CUDA is available, we have found  1  GPU(s)\n",
+      "NVIDIA GeForce RTX 4080 Laptop GPU\n",
+      "CUDA version: 12.1\n",
+      "hf_embeddings_device_type: cuda:0\n",
+      "hf_pipeline_device_type: cuda:0\n",
+      "load INSTRUCTOR_Transformer\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n",
+      "max_seq_length  512\n",
+      "Completed in 0.554s\n",
+      "Load index from ./data/faiss_1024_512/ with FAISS\n",
+      "Completed in 0.018s\n",
+      "initializing LLM: hftgi\n",
+      "  hf_pipeline_device_type: cuda:0\n",
+      "     load_quantized_model: None\n",
+      "              torch_dtype: torch.float16\n",
+      "                 n_threds: 4\n",
+      "       repetition_penalty: 1.05\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/pydantic/_internal/_fields.py:149: UserWarning: Field \"model_id\" has conflict with protected namespace \"model_\".\n",
+      "\n",
+      "You may be able to resolve this warning by setting `model_config['protected_namespaces'] = ()`.\n",
+      "  warnings.warn(\n",
+      "initialization complete\n",
+      "Completed in 0.011s\n",
+      "\n",
+      "Question: What's PCI DSS?\n",
+      "{'question': \"What's PCI DSS?\", 'chat_history': []}\n",
+      " PCI DSS is a global standard that provides a baseline of technical and operational requirements designed to protect payment account data. It helps ensure the security of cardholder data and reduces the risk of cyberattacks.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 4.609s\n",
+      "\n",
+      "Question: Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?\n",
+      "{'question': 'Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS is a global standard that provides a baseline of technical and operational requirements designed to protect payment account data. It helps ensure the security of cardholder data and reduces the risk of cyberattacks.')]}\n",
+      " ¿Puedes resumir los cambios realizados desde la versión 3.2.1 hasta la versión 4.0 de PCI DSS?\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " Sí, puedo resumir los cambios realizados desde la versión 3.2.1 hasta la versión 4.0 de PCI DSS. Los cambios incluyen ajustes para alinear con las modificaciones entre las versiones 3.2 y 3.2.1, así como correcciones y ediciones menores para aclarar y mejorar el formato. Además, hay cambios en las secciones introductorias, general y adicionales de los requisitos.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 9.076s\n",
+      "\n",
+      "Question: new requirements for vulnerability assessments\n",
+      "{'question': 'new requirements for vulnerability assessments', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS is a global standard that provides a baseline of technical and operational requirements designed to protect payment account data. It helps ensure the security of cardholder data and reduces the risk of cyberattacks.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'Sí, puedo resumir los cambios realizados desde la versión 3.2.1 hasta la versión 4.0 de PCI DSS. Los cambios incluyen ajustes para alinear con las modificaciones entre las versiones 3.2 y 3.2.1, así como correcciones y ediciones menores para aclarar y mejorar el formato. Además, hay cambios en las secciones introductorias, general y adicionales de los requisitos.')]}\n",
+      " ¿Cuáles son las nuevas recomendaciones para las evaluaciones de vulnerabilidades en la versión 4.0 de PCI DSS?\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "\n",
+      "Las nuevas recomendaciones para las evaluaciones de vulnerabilidades en la versión 4.0 de PCI DSS incluyen realizar escaneos de vulnerabilidad externa después de cualquier cambio significativo y resolver las vulnerabilidades con una puntuación de 4.0 o superior según el CVSS. Además, se recomienda realizar pruebas de penetración externa e interna regularmente y corregir las vulnerabilidades y debilidades de seguridad que sean explotables.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 9.533s\n",
+      "\n",
+      "Question: more on penetration testing\n",
+      "{'question': 'more on penetration testing', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS is a global standard that provides a baseline of technical and operational requirements designed to protect payment account data. It helps ensure the security of cardholder data and reduces the risk of cyberattacks.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'Sí, puedo resumir los cambios realizados desde la versión 3.2.1 hasta la versión 4.0 de PCI DSS. Los cambios incluyen ajustes para alinear con las modificaciones entre las versiones 3.2 y 3.2.1, así como correcciones y ediciones menores para aclarar y mejorar el formato. Además, hay cambios en las secciones introductorias, general y adicionales de los requisitos.'), ('new requirements for vulnerability assessments', 'Las nuevas recomendaciones para las evaluaciones de vulnerabilidades en la versión 4.0 de PCI DSS incluyen realizar escaneos de vulnerabilidad externa después de cualquier cambio significativo y resolver las vulnerabilidades con una puntuación de 4.0 o superior según el CVSS. Además, se recomienda realizar pruebas de penetración externa e interna regularmente y corregir las vulnerabilidades y debilidades de seguridad que sean explotables.')]}\n",
+      " ¿Puedes dar más información sobre las pruebas de penetración?\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " Sí, puedo dar más información sobre las pruebas de penetración. Para obtener más información, consulte la documentación y realice entrevistas con el personal involucrado en la implementación y ejecución de las pruebas de penetración. Asegúrese de identificar y documentar los números de referencia de la evidencia correspondientes a las secciones 6 para todas las entrevistas y documentaciones examinadas.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 7.629s\n",
+      "Total time used: 30.847 s\n",
+      "Number of tokens generated: 455\n",
+      "Average generation speed: 14.750 tokens/s\n",
+      "CPU times: user 651 ms, sys: 120 ms, total: 771 ms\n",
+      "Wall time: 34.5 s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "os.environ[\"HFTGI_RP\"] = \"1.05\"\n",
+    "!python qa_chain_test.py"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "8a022a7b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading env vars from: /home/inflaton/code/projects/chat-with-orca-2/.env\n",
+      "Running on:  Linux-5.10.102.1-microsoft-standard-WSL2-x86_64-with-glibc2.35\n",
+      "MPS is NOT available\n",
+      "CUDA is  available\n",
+      "MPS not available because the current PyTorch install was not built with MPS enabled.\n",
+      "CUDA is available, we have found  1  GPU(s)\n",
+      "NVIDIA GeForce RTX 4080 Laptop GPU\n",
+      "CUDA version: 12.1\n",
+      "hf_embeddings_device_type: cuda:0\n",
+      "hf_pipeline_device_type: cuda:0\n",
+      "load INSTRUCTOR_Transformer\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n",
+      "max_seq_length  512\n",
+      "Completed in 0.589s\n",
+      "Load index from ./data/faiss_1024_512/ with FAISS\n",
+      "Completed in 0.019s\n",
+      "initializing LLM: hftgi\n",
+      "  hf_pipeline_device_type: cuda:0\n",
+      "     load_quantized_model: None\n",
+      "              torch_dtype: torch.float16\n",
+      "                 n_threds: 4\n",
+      "       repetition_penalty: 1.1\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/pydantic/_internal/_fields.py:149: UserWarning: Field \"model_id\" has conflict with protected namespace \"model_\".\n",
+      "\n",
+      "You may be able to resolve this warning by setting `model_config['protected_namespaces'] = ()`.\n",
+      "  warnings.warn(\n",
+      "initialization complete\n",
+      "Completed in 0.011s\n",
+      "\n",
+      "Question: What's PCI DSS?\n",
+      "{'question': \"What's PCI DSS?\", 'chat_history': []}\n",
+      " PCI DSS is a set of requirements designed to protect payment account data. It provides a baseline of technical and operational requirements for organizations that handle payment card transactions.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 4.059s\n",
+      "\n",
+      "Question: Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?\n",
+      "{'question': 'Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS is a set of requirements designed to protect payment account data. It provides a baseline of technical and operational requirements for organizations that handle payment card transactions.')]}\n",
+      " ¿Puedes resumir los cambios realizados desde la versión 3.2.1 hasta la versión 4.0 de PCI DSS?\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " Sí, puedo resumir los cambios realizados desde la versión 3.2.1 hasta la versión 4.0 de PCI DSS. Los cambios incluyen ajustes para alinear con las modificaciones entre las versiones 3.2 y 3.2.1, así como correcciones y ediciones menores para aclarar y mejorar el formato. Además, hay actualizaciones para adaptarse a las nuevas exigencias y mejoras en la seguridad.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 9.049s\n",
+      "\n",
+      "Question: new requirements for vulnerability assessments\n",
+      "{'question': 'new requirements for vulnerability assessments', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS is a set of requirements designed to protect payment account data. It provides a baseline of technical and operational requirements for organizations that handle payment card transactions.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'Sí, puedo resumir los cambios realizados desde la versión 3.2.1 hasta la versión 4.0 de PCI DSS. Los cambios incluyen ajustes para alinear con las modificaciones entre las versiones 3.2 y 3.2.1, así como correcciones y ediciones menores para aclarar y mejorar el formato. Además, hay actualizaciones para adaptarse a las nuevas exigencias y mejoras en la seguridad.')]}\n",
+      " ¿Cuáles son las nuevas exigencias para las evaluaciones de vulnerabilidades en la versión 4.0 de PCI DSS?\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " Para obtener información sobre las nuevas exigencias para las evaluaciones de vulnerabilidades en la versión 4.0 de PCI DSS, consulte la sección \"Descripción del cambio\" en el documento \"PCI DSS v3.2.1 a v.4.0 Resumen de cambios r2 diciembre 2022\".\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 7.425s\n",
+      "\n",
+      "Question: more on penetration testing\n",
+      "{'question': 'more on penetration testing', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS is a set of requirements designed to protect payment account data. It provides a baseline of technical and operational requirements for organizations that handle payment card transactions.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'Sí, puedo resumir los cambios realizados desde la versión 3.2.1 hasta la versión 4.0 de PCI DSS. Los cambios incluyen ajustes para alinear con las modificaciones entre las versiones 3.2 y 3.2.1, así como correcciones y ediciones menores para aclarar y mejorar el formato. Además, hay actualizaciones para adaptarse a las nuevas exigencias y mejoras en la seguridad.'), ('new requirements for vulnerability assessments', 'Para obtener información sobre las nuevas exigencias para las evaluaciones de vulnerabilidades en la versión 4.0 de PCI DSS, consulte la sección \"Descripción del cambio\" en el documento \"PCI DSS v3.2.1 a v.4.0 Resumen de cambios r2 diciembre 2022\".')]}\n",
+      " ¿Podría proporcionar más información sobre las pruebas de penetración en el contexto de PCI DSS?\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " Sí, puedo proporcionar más información sobre las pruebas de penetración en el contexto de PCI DSS.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 3.957s\n",
+      "Total time used: 24.490 s\n",
+      "Number of tokens generated: 348\n",
+      "Average generation speed: 14.210 tokens/s\n",
+      "CPU times: user 434 ms, sys: 177 ms, total: 612 ms\n",
+      "Wall time: 28.2 s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "os.environ[\"HFTGI_RP\"] = \"1.10\"\n",
+    "!python qa_chain_test.py"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "db48dcc4",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading env vars from: /home/inflaton/code/projects/chat-with-orca-2/.env\n",
+      "Running on:  Linux-5.10.102.1-microsoft-standard-WSL2-x86_64-with-glibc2.35\n",
+      "MPS is NOT available\n",
+      "CUDA is  available\n",
+      "MPS not available because the current PyTorch install was not built with MPS enabled.\n",
+      "CUDA is available, we have found  1  GPU(s)\n",
+      "NVIDIA GeForce RTX 4080 Laptop GPU\n",
+      "CUDA version: 12.1\n",
+      "hf_embeddings_device_type: cuda:0\n",
+      "hf_pipeline_device_type: cuda:0\n",
+      "load INSTRUCTOR_Transformer\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n",
+      "max_seq_length  512\n",
+      "Completed in 0.583s\n",
+      "Load index from ./data/faiss_1024_512/ with FAISS\n",
+      "Completed in 0.019s\n",
+      "initializing LLM: hftgi\n",
+      "  hf_pipeline_device_type: cuda:0\n",
+      "     load_quantized_model: None\n",
+      "              torch_dtype: torch.float16\n",
+      "                 n_threds: 4\n",
+      "       repetition_penalty: 1.15\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/pydantic/_internal/_fields.py:149: UserWarning: Field \"model_id\" has conflict with protected namespace \"model_\".\n",
+      "\n",
+      "You may be able to resolve this warning by setting `model_config['protected_namespaces'] = ()`.\n",
+      "  warnings.warn(\n",
+      "initialization complete\n",
+      "Completed in 0.011s\n",
+      "\n",
+      "Question: What's PCI DSS?\n",
+      "{'question': \"What's PCI DSS?\", 'chat_history': []}\n",
+      " PCI DSS is a set of requirements designed to protect payment account data. It includes guidelines for building and maintaining secure networks and systems, protecting account data, managing vulnerabilities, implementing strong access control measures, and more.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 4.767s\n",
+      "\n",
+      "Question: Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?\n",
+      "{'question': 'Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS is a set of requirements designed to protect payment account data. It includes guidelines for building and maintaining secure networks and systems, protecting account data, managing vulnerabilities, implementing strong access control measures, and more.')]}\n",
+      " ¿Puedes resumir los cambios realizados desde la versión 3.2.1 hasta la versión 4.0 de PCI DSS?\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " Sí, puedo resumir los cambios realizados desde la versión 3.2.1 hasta la versión 4.0 de PCI DSS. Los cambios incluyen ajustes para alinear con las modificaciones entre las versiones 3.2 y 3.2.1, así como correcciones menores y ediciones para mejorar la claridad y el formato. Además, hay cambios en las secciones introductorias, generalidades y específicas de cada requisito. Por ejemplo, se ha eliminado la opción de \"In Place with Remediation\" como una opción de informe.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 10.812s\n",
+      "\n",
+      "Question: new requirements for vulnerability assessments\n",
+      "{'question': 'new requirements for vulnerability assessments', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS is a set of requirements designed to protect payment account data. It includes guidelines for building and maintaining secure networks and systems, protecting account data, managing vulnerabilities, implementing strong access control measures, and more.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'Sí, puedo resumir los cambios realizados desde la versión 3.2.1 hasta la versión 4.0 de PCI DSS. Los cambios incluyen ajustes para alinear con las modificaciones entre las versiones 3.2 y 3.2.1, así como correcciones menores y ediciones para mejorar la claridad y el formato. Además, hay cambios en las secciones introductorias, generalidades y específicas de cada requisito. Por ejemplo, se ha eliminado la opción de \"In Place with Remediation\" como una opción de informe.')]}\n",
+      " ¿Cuáles son las nuevas recomendaciones para las evaluaciones de vulnerabilidades en la versión 4.0 de PCI DSS?\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " Some possible helpful answers are:\n",
+      "\n",
+      "- Las nuevas recomendaciones incluyen realizar escanes de vulnerabilidad externa después de cualquier cambio significativo y resolver las vulnerabilidades con una puntuación de 4.0 o superior por el CVSS. También se recomienda realizar rescans cuando sea necesario y contratar personal calificado para realizar los escaneos.\n",
+      "- Según la versión 4.0 de PCI DSS, las nuevas recomendaciones para las evaluaciones de vulnerabilidades son: realizar escanes de vulnerabilidad externa después de cada cambio importante, resolver las vulnerabilidades con un puntaje de 4.0 o más en el CVSS, y hacer rescans adicionales si es necesario. Además, se debe contar con personal capacitado para llevar a cabo los escaneos.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 14.112s\n",
+      "\n",
+      "Question: more on penetration testing\n",
+      "{'question': 'more on penetration testing', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS is a set of requirements designed to protect payment account data. It includes guidelines for building and maintaining secure networks and systems, protecting account data, managing vulnerabilities, implementing strong access control measures, and more.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'Sí, puedo resumir los cambios realizados desde la versión 3.2.1 hasta la versión 4.0 de PCI DSS. Los cambios incluyen ajustes para alinear con las modificaciones entre las versiones 3.2 y 3.2.1, así como correcciones menores y ediciones para mejorar la claridad y el formato. Además, hay cambios en las secciones introductorias, generalidades y específicas de cada requisito. Por ejemplo, se ha eliminado la opción de \"In Place with Remediation\" como una opción de informe.'), ('new requirements for vulnerability assessments', 'Some possible helpful answers are:\\n\\n- Las nuevas recomendaciones incluyen realizar escanes de vulnerabilidad externa después de cualquier cambio significativo y resolver las vulnerabilidades con una puntuación de 4.0 o superior por el CVSS. También se recomienda realizar rescans cuando sea necesario y contratar personal calificado para realizar los escaneos.\\n- Según la versión 4.0 de PCI DSS, las nuevas recomendaciones para las evaluaciones de vulnerabilidades son: realizar escanes de vulnerabilidad externa después de cada cambio importante, resolver las vulnerabilidades con un puntaje de 4.0 o más en el CVSS, y hacer rescans adicionales si es necesario. Además, se debe contar con personal capacitado para llevar a cabo los escaneos.')]}\n",
+      " ¿Cuál es el papel del prueba de penetración en el proceso de seguridad de PCI DSS?\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " La prueba de penetración es una parte importante del proceso de seguridad de PCI DSS. Se utiliza para identificar vulnerabilidades y debilidades de seguridad que podrían ser explotadas por atacantes malintencionados. Según la documentación proporcionada, la prueba de penetración debe ser realizada al menos una vez cada doce meses, después de cualquier modificación o cambio significativo en la infraestructura o aplicaciones, y por un recurso interno calificado o tercera parte externa calificada. Además, las vulnerabilidades y debilidades de seguridad que se encuentren durante la prueba de penetración deben ser corregidas y verificadas mediante otra prueba de penetración.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 12.884s\n",
+      "Total time used: 42.575 s\n",
+      "Number of tokens generated: 649\n",
+      "Average generation speed: 15.244 tokens/s\n",
+      "CPU times: user 677 ms, sys: 321 ms, total: 998 ms\n",
+      "Wall time: 46.3 s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "os.environ[\"HFTGI_RP\"] = \"1.15\"\n",
+    "!python qa_chain_test.py"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebook/nvidia-4080-orca-2-7b.ipynb

@@ -0,0 +1,435 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "874cd58e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "workding dir: /home/inflaton/code/projects/chat-with-orca-2\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import sys\n",
+    "from pathlib import Path\n",
+    "\n",
+    "workding_dir = str(Path.cwd().parent)\n",
+    "os.chdir(workding_dir)\n",
+    "sys.path.append(workding_dir)\n",
+    "print(\"workding dir:\", workding_dir)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "19f967ca",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "os.environ[\"LLM_MODEL_TYPE\"] = \"huggingface\"\n",
+    "os.environ[\"HUGGINGFACE_MODEL_NAME_OR_PATH\"] = \"microsoft/Orca-2-7b\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "a6d96660",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading env vars from: /home/inflaton/code/projects/chat-with-orca-2/.env\n",
+      "Running on:  Linux-5.10.102.1-microsoft-standard-WSL2-x86_64-with-glibc2.35\n",
+      "MPS is NOT available\n",
+      "CUDA is  available\n",
+      "MPS not available because the current PyTorch install was not built with MPS enabled.\n",
+      "CUDA is available, we have found  1  GPU(s)\n",
+      "NVIDIA GeForce RTX 4080 Laptop GPU\n",
+      "CUDA version: 12.1\n",
+      "hf_embeddings_device_type: cuda:0\n",
+      "hf_pipeline_device_type: cuda:0\n",
+      "load INSTRUCTOR_Transformer\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n",
+      "max_seq_length  512\n",
+      "Completed in 0.793s\n",
+      "Load index from ./data/faiss_1024_512/ with FAISS\n",
+      "Completed in 0.025s\n",
+      "initializing LLM: huggingface\n",
+      "  hf_pipeline_device_type: cuda:0\n",
+      "     load_quantized_model: None\n",
+      "              torch_dtype: torch.float16\n",
+      "                 n_threds: 4\n",
+      "            loading model: microsoft/Orca-2-7b\n",
+      "            HF auth token: None\n",
+      "       repetition_penalty: 1.05\n",
+      "Loading checkpoint shards: 100%|█████████████████| 3/3 [06:32<00:00, 130.71s/it]\n",
+      "Model memory footprint: 13543997440\n",
+      "initialization complete\n",
+      "Completed in 399.575s\n",
+      "\n",
+      "Question: What's PCI DSS?\n",
+      "{'question': \"What's PCI DSS?\", 'chat_history': []}\n",
+      "PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment data.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 32.938s\n",
+      "\n",
+      "Question: Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?\n",
+      "{'question': 'Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment data.')]}\n",
+      "What are the main differences between PCI DSS version 3.2.1 and version 4.0?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "The main differences between PCI DSS version 3.2.1 and version 4.0 include changes to the requirements, testing procedures, and guidance. The standard has been updated to address new risks and vulnerabilities, as well as to reflect advances in technology and payment card industry practices. Some specific changes include updates to Requirement 6.3.3, the addition of a description for the change made to Requirement 8.3.9, and the addition of a description of the change made to Requirement 3.6.1.1. Additionally, the standard has been revised to align with the changes from PCI DSS v3.2 to PCI DSS v3.2.1 and to incorporate minor corrections and edits made for clarification and/or format purposes.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 80.754s\n",
+      "\n",
+      "Question: new requirements for vulnerability assessments\n",
+      "{'question': 'new requirements for vulnerability assessments', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment data.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'The main differences between PCI DSS version 3.2.1 and version 4.0 include changes to the requirements, testing procedures, and guidance. The standard has been updated to address new risks and vulnerabilities, as well as to reflect advances in technology and payment card industry practices. Some specific changes include updates to Requirement 6.3.3, the addition of a description for the change made to Requirement 8.3.9, and the addition of a description of the change made to Requirement 3.6.1.1. Additionally, the standard has been revised to align with the changes from PCI DSS v3.2 to PCI DSS v3.2.1 and to incorporate minor corrections and edits made for clarification and/or format purposes.')]}\n",
+      "What are the new requirements for vulnerability assessments in PCI DSS version 4.0?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "The new requirements for vulnerability assessments in PCI DSS version 4.0 include:\n",
+      "\n",
+      "- Requirement 4.1.2, which states that there is a new requirement for roles and responsibilities related to vulnerability assessments.\n",
+      "- Requirement 4.2.1, which requires confirmation that certificates used for Point-to-Point (PAN) transmissions over open, public networks are valid and not expired or revoked. This bullet is a best practice until March 31, 2025.\n",
+      "- Requirement 4.2.1.1, which introduces a new requirement to maintain an inventory of trusted keys and certificates. This is also a best practice until March 31, 2025.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 90.698s\n",
+      "\n",
+      "Question: more on penetration testing\n",
+      "{'question': 'more on penetration testing', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment data.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'The main differences between PCI DSS version 3.2.1 and version 4.0 include changes to the requirements, testing procedures, and guidance. The standard has been updated to address new risks and vulnerabilities, as well as to reflect advances in technology and payment card industry practices. Some specific changes include updates to Requirement 6.3.3, the addition of a description for the change made to Requirement 8.3.9, and the addition of a description of the change made to Requirement 3.6.1.1. Additionally, the standard has been revised to align with the changes from PCI DSS v3.2 to PCI DSS v3.2.1 and to incorporate minor corrections and edits made for clarification and/or format purposes.'), ('new requirements for vulnerability assessments', 'The new requirements for vulnerability assessments in PCI DSS version 4.0 include:\\n\\n- Requirement 4.1.2, which states that there is a new requirement for roles and responsibilities related to vulnerability assessments.\\n- Requirement 4.2.1, which requires confirmation that certificates used for Point-to-Point (PAN) transmissions over open, public networks are valid and not expired or revoked. This bullet is a best practice until March 31, 2025.\\n- Requirement 4.2.1.1, which introduces a new requirement to maintain an inventory of trusted keys and certificates. This is also a best practice until March 31, 2025.')]}\n",
+      "What are the new requirements for penetration testing in PCI DSS version 4.0?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "The new requirements for penetration testing in PCI DSS version 4.0 include defining a penetration testing methodology, covering the entire CDE perimeter and critical systems, testing from both inside and outside the network, testing to validate any segmentation and scope-reduction controls, and performing application-layer and network-layer penetration testing.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 50.591s\n",
+      "Total time used: 254.981 s\n",
+      "Number of tokens generated: 536\n",
+      "Average generation speed: 2.102 tokens/s\n",
+      "CPU times: user 9.03 s, sys: 2.26 s, total: 11.3 s\n",
+      "Wall time: 11min\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "os.environ[\"HF_RP\"] = \"1.05\"\n",
+    "!python qa_chain_test.py"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "8a022a7b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading env vars from: /home/inflaton/code/projects/chat-with-orca-2/.env\n",
+      "Running on:  Linux-5.10.102.1-microsoft-standard-WSL2-x86_64-with-glibc2.35\n",
+      "MPS is NOT available\n",
+      "CUDA is  available\n",
+      "MPS not available because the current PyTorch install was not built with MPS enabled.\n",
+      "CUDA is available, we have found  1  GPU(s)\n",
+      "NVIDIA GeForce RTX 4080 Laptop GPU\n",
+      "CUDA version: 12.1\n",
+      "hf_embeddings_device_type: cuda:0\n",
+      "hf_pipeline_device_type: cuda:0\n",
+      "load INSTRUCTOR_Transformer\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n",
+      "max_seq_length  512\n",
+      "Completed in 0.995s\n",
+      "Load index from ./data/faiss_1024_512/ with FAISS\n",
+      "Completed in 0.033s\n",
+      "initializing LLM: huggingface\n",
+      "  hf_pipeline_device_type: cuda:0\n",
+      "     load_quantized_model: None\n",
+      "              torch_dtype: torch.float16\n",
+      "                 n_threds: 4\n",
+      "            loading model: microsoft/Orca-2-7b\n",
+      "            HF auth token: None\n",
+      "       repetition_penalty: 1.1\n",
+      "Loading checkpoint shards: 100%|█████████████████| 3/3 [05:36<00:00, 112.28s/it]\n",
+      "Model memory footprint: 13543997440\n",
+      "initialization complete\n",
+      "Completed in 345.225s\n",
+      "\n",
+      "Question: What's PCI DSS?\n",
+      "{'question': \"What's PCI DSS?\", 'chat_history': []}\n",
+      "PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment data.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 34.481s\n",
+      "\n",
+      "Question: Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?\n",
+      "{'question': 'Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment data.')]}\n",
+      "What are the main differences between PCI DSS versions 3.2.1 and 4.0?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "The main differences between PCI DSS versions 3.2.1 and 4.0 include changes to the introductory sections, general changes to the requirements, testing procedures, and guidance, and additional changes per requirement.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 38.270s\n",
+      "\n",
+      "Question: new requirements for vulnerability assessments\n",
+      "{'question': 'new requirements for vulnerability assessments', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment data.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'The main differences between PCI DSS versions 3.2.1 and 4.0 include changes to the introductory sections, general changes to the requirements, testing procedures, and guidance, and additional changes per requirement.')]}\n",
+      "What are the new requirements for vulnerability assessments in PCI DSS version 4.0?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "There are several new requirements for vulnerability assessments in PCI DSS version 4.0. Some of them include:\n",
+      "\n",
+      "1. Requirement 4.1.2, which states that there should be clear roles and responsibilities defined for vulnerability management.\n",
+      "2. Requirement 4.2.1, which requires organizations to ensure that certificates used for point-to-point encryption (PAN) transmissions over open, public networks are valid and not expired or revoked. This is considered a best practice until March 31, 2025.\n",
+      "3. Requirement 10.4.2.1, which mandates a targeted risk analysis to determine the frequency of log reviews for all other system components.\n",
+      "4. Requirement 10.7.2 and 10.7.3, which require organizations to detect, alert, and address failures of critical security control systems promptly.\n",
+      "5. Requirement 11.3.1.1 and 11.3.1.2, which require internal vulnerability scans to be performed using authenticated scanning and to manage all applicable vulnerabilities, including those not ranked as high-risk or critical.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 134.464s\n",
+      "\n",
+      "Question: more on penetration testing\n",
+      "{'question': 'more on penetration testing', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment data.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'The main differences between PCI DSS versions 3.2.1 and 4.0 include changes to the introductory sections, general changes to the requirements, testing procedures, and guidance, and additional changes per requirement.'), ('new requirements for vulnerability assessments', 'There are several new requirements for vulnerability assessments in PCI DSS version 4.0. Some of them include:\\n\\n1. Requirement 4.1.2, which states that there should be clear roles and responsibilities defined for vulnerability management.\\n2. Requirement 4.2.1, which requires organizations to ensure that certificates used for point-to-point encryption (PAN) transmissions over open, public networks are valid and not expired or revoked. This is considered a best practice until March 31, 2025.\\n3. Requirement 10.4.2.1, which mandates a targeted risk analysis to determine the frequency of log reviews for all other system components.\\n4. Requirement 10.7.2 and 10.7.3, which require organizations to detect, alert, and address failures of critical security control systems promptly.\\n5. Requirement 11.3.1.1 and 11.3.1.2, which require internal vulnerability scans to be performed using authenticated scanning and to manage all applicable vulnerabilities, including those not ranked as high-risk or critical.')]}\n",
+      "What are some key differences between penetration testing and vulnerability assessments in the context of PCI DSS?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Penetration testing and vulnerability assessments are both important security measures in the context of PCI DSS, but they differ in their focus and approach.\n",
+      "\n",
+      "Penetration testing involves simulating attacks by an external or internal attacker to identify vulnerabilities and exploit them, while vulnerability assessments involve scanning and analyzing systems and networks to identify potential weaknesses and risks. Penetration testing is more focused on detecting and exploiting vulnerabilities, whereas vulnerability assessments are more focused on identifying and mitigating risks.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 66.303s\n",
+      "Total time used: 273.518 s\n",
+      "Number of tokens generated: 549\n",
+      "Average generation speed: 2.007 tokens/s\n",
+      "CPU times: user 7.99 s, sys: 2.19 s, total: 10.2 s\n",
+      "Wall time: 10min 25s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "os.environ[\"HF_RP\"] = \"1.10\"\n",
+    "!python qa_chain_test.py"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "db48dcc4",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading env vars from: /home/inflaton/code/projects/chat-with-orca-2/.env\n",
+      "Running on:  Linux-5.10.102.1-microsoft-standard-WSL2-x86_64-with-glibc2.35\n",
+      "MPS is NOT available\n",
+      "CUDA is  available\n",
+      "MPS not available because the current PyTorch install was not built with MPS enabled.\n",
+      "CUDA is available, we have found  1  GPU(s)\n",
+      "NVIDIA GeForce RTX 4080 Laptop GPU\n",
+      "CUDA version: 12.1\n",
+      "hf_embeddings_device_type: cuda:0\n",
+      "hf_pipeline_device_type: cuda:0\n",
+      "load INSTRUCTOR_Transformer\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n",
+      "max_seq_length  512\n",
+      "Completed in 0.611s\n",
+      "Load index from ./data/faiss_1024_512/ with FAISS\n",
+      "Completed in 0.020s\n",
+      "initializing LLM: huggingface\n",
+      "  hf_pipeline_device_type: cuda:0\n",
+      "     load_quantized_model: None\n",
+      "              torch_dtype: torch.float16\n",
+      "                 n_threds: 4\n",
+      "            loading model: microsoft/Orca-2-7b\n",
+      "            HF auth token: None\n",
+      "       repetition_penalty: 1.15\n",
+      "Loading checkpoint shards: 100%|█████████████████| 3/3 [05:53<00:00, 117.86s/it]\n",
+      "Model memory footprint: 13543997440\n",
+      "initialization complete\n",
+      "Completed in 359.955s\n",
+      "\n",
+      "Question: What's PCI DSS?\n",
+      "{'question': \"What's PCI DSS?\", 'chat_history': []}\n",
+      "PCI DSS stands for Payment Card Industry Data Security Standard. It is a set of technical and operational requirements designed to protect payment data.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 29.134s\n",
+      "\n",
+      "Question: Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?\n",
+      "{'question': 'Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a set of technical and operational requirements designed to protect payment data.')]}\n",
+      "How does PCI DSS version 4.0 differ from version 3.2.1 in terms of security standards?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "In terms of security standards, PCI DSS version 4.0 has several differences compared to version 3.2.1. Some of these include new or revised requirements related to data encryption, multi-factor authentication, secure software development, and network segmentation. Additionally, there are updated guidelines for penetration testing and quarterly vulnerability scans.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 42.657s\n",
+      "\n",
+      "Question: new requirements for vulnerability assessments\n",
+      "{'question': 'new requirements for vulnerability assessments', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a set of technical and operational requirements designed to protect payment data.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'In terms of security standards, PCI DSS version 4.0 has several differences compared to version 3.2.1. Some of these include new or revised requirements related to data encryption, multi-factor authentication, secure software development, and network segmentation. Additionally, there are updated guidelines for penetration testing and quarterly vulnerability scans.')]}\n",
+      "What are the new requirements for vulnerability assessments in PCI DSS version 4.0?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "There are several new requirements related to vulnerability assessments in PCI DSS version 4.0. Some examples include:\n",
+      "\n",
+      "- Requirement 4.1.2, which states that there should be clear roles and responsibilities defined for handling vulnerability assessments.\n",
+      "- Requirement 4.2.1, which requires organizations to ensure that certificates used for point-to-point encryption (PAN) transmissions over open, public networks are valid and not expired or revoked. This bullet is considered a best practice until March 31, 2025.\n",
+      "- Requirement 10.4.2.1, which mandates a targeted risk analysis to determine how often logs should be reviewed for all other system components.\n",
+      "- Requirement 10.7.2 and 10.7.3, which emphasize the importance of detecting, alerting, and addressing failures of critical security control systems promptly.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 110.695s\n",
+      "\n",
+      "Question: more on penetration testing\n",
+      "{'question': 'more on penetration testing', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a set of technical and operational requirements designed to protect payment data.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'In terms of security standards, PCI DSS version 4.0 has several differences compared to version 3.2.1. Some of these include new or revised requirements related to data encryption, multi-factor authentication, secure software development, and network segmentation. Additionally, there are updated guidelines for penetration testing and quarterly vulnerability scans.'), ('new requirements for vulnerability assessments', 'There are several new requirements related to vulnerability assessments in PCI DSS version 4.0. Some examples include:\\n\\n- Requirement 4.1.2, which states that there should be clear roles and responsibilities defined for handling vulnerability assessments.\\n- Requirement 4.2.1, which requires organizations to ensure that certificates used for point-to-point encryption (PAN) transmissions over open, public networks are valid and not expired or revoked. This bullet is considered a best practice until March 31, 2025.\\n- Requirement 10.4.2.1, which mandates a targeted risk analysis to determine how often logs should be reviewed for all other system components.\\n- Requirement 10.7.2 and 10.7.3, which emphasize the importance of detecting, alerting, and addressing failures of critical security control systems promptly.')]}\n",
+      "What are some specific techniques used during penetration testing to identify potential vulnerabilities in a system?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Some specific techniques used during penetration testing include scanning networks, identifying open ports, analyzing code for vulnerabilities, and performing social engineering attacks.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 31.630s\n",
+      "Total time used: 214.116 s\n",
+      "Number of tokens generated: 432\n",
+      "Average generation speed: 2.018 tokens/s\n",
+      "CPU times: user 7.37 s, sys: 1.95 s, total: 9.32 s\n",
+      "Wall time: 9min 38s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "os.environ[\"HF_RP\"] = \"1.15\"\n",
+    "!python qa_chain_test.py"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebook/nvidia-4080-orca-2-7b_tgi.ipynb

@@ -0,0 +1,369 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "874cd58e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "workding dir: /home/inflaton/code/projects/chat-with-orca-2\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import sys\n",
+    "from pathlib import Path\n",
+    "\n",
+    "workding_dir = str(Path.cwd().parent)\n",
+    "os.chdir(workding_dir)\n",
+    "sys.path.append(workding_dir)\n",
+    "print(\"workding dir:\", workding_dir)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "19f967ca",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "os.environ[\"LLM_MODEL_TYPE\"] = \"hftgi\"\n",
+    "os.environ[\"HFTGI_SERVER_URL\"] = \"https://legible-illegally-monarch.ngrok-free.app\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "a6d96660",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading env vars from: /home/inflaton/code/projects/chat-with-orca-2/.env\n",
+      "Running on:  Linux-5.10.102.1-microsoft-standard-WSL2-x86_64-with-glibc2.35\n",
+      "MPS is NOT available\n",
+      "CUDA is  available\n",
+      "MPS not available because the current PyTorch install was not built with MPS enabled.\n",
+      "CUDA is available, we have found  1  GPU(s)\n",
+      "NVIDIA GeForce RTX 4080 Laptop GPU\n",
+      "CUDA version: 12.1\n",
+      "hf_embeddings_device_type: cuda:0\n",
+      "hf_pipeline_device_type: cuda:0\n",
+      "load INSTRUCTOR_Transformer\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n",
+      "max_seq_length  512\n",
+      "Completed in 1.255s\n",
+      "Load index from ./data/faiss_1024_512/ with FAISS\n",
+      "Completed in 0.041s\n",
+      "initializing LLM: hftgi\n",
+      "  hf_pipeline_device_type: cuda:0\n",
+      "     load_quantized_model: None\n",
+      "              torch_dtype: torch.float16\n",
+      "                 n_threds: 4\n",
+      "       repetition_penalty: 1.05\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/pydantic/_internal/_fields.py:149: UserWarning: Field \"model_id\" has conflict with protected namespace \"model_\".\n",
+      "\n",
+      "You may be able to resolve this warning by setting `model_config['protected_namespaces'] = ()`.\n",
+      "  warnings.warn(\n",
+      "initialization complete\n",
+      "Completed in 0.011s\n",
+      "\n",
+      "Question: What's PCI DSS?\n",
+      "{'question': \"What's PCI DSS?\", 'chat_history': []}\n",
+      " PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment data.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 34.576s\n",
+      "\n",
+      "Question: Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?\n",
+      "{'question': 'Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment data.')]}\n",
+      " What are the key differences between PCI DSS version 3.2.1 and version 4.0?\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " The key differences between PCI DSS version 3.2.1 and version 4.0 include changes to the requirements, testing procedures, and guidance. The standard should be reviewed in its entirety rather than focusing solely on this summary document.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 44.635s\n",
+      "\n",
+      "Question: new requirements for vulnerability assessments\n",
+      "{'question': 'new requirements for vulnerability assessments', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment data.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'The key differences between PCI DSS version 3.2.1 and version 4.0 include changes to the requirements, testing procedures, and guidance. The standard should be reviewed in its entirety rather than focusing solely on this summary document.')]}\n",
+      " What are the new requirements for vulnerability assessments in PCI DSS version 4.0?\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " The new requirements for vulnerability assessments in PCI DSS version 4.0 include:\n",
+      "\n",
+      "1. Requirement 4.1.2, which states that there is a new requirement for roles and responsibilities related to vulnerability assessments.\n",
+      "2. Requirement 4.2.1, which requires organizations to confirm that certificates used for Point-to-Point Network (PAN) transmissions over open, public networks are valid and not expired or revoked. This bullet is a best practice until March 31, 2025.\n",
+      "3. Requirement 10.4.2.1, which requires a targeted risk analysis to determine the frequency of log reviews for all other system components.\n",
+      "4. Requirement 10.7.2 and 10.7.3, which require organizations to detect, alert, and address failures of critical security control systems promptly.\n",
+      "5. Requirement 11.3.1.1 and 11.3.1.2, which require organizations to manage all applicable vulnerabilities (not ranked as high-risk or critical) and perform internal vulnerability scans via authenticated scanning.\n",
+      "6. Requirement A1.2.3, which requires multi-tenant service providers to implement processes or mechanisms for reporting and addressing suspected or confirmed security incidents and vulnerabilities.\n",
+      "7. Requirement A3.3.1, which requires multi-tenant service providers to detect, alert, and report failures of automated log review mechanisms and automated code review tools in a timely manner.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 188.142s\n",
+      "\n",
+      "Question: more on penetration testing\n",
+      "{'question': 'more on penetration testing', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment data.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'The key differences between PCI DSS version 3.2.1 and version 4.0 include changes to the requirements, testing procedures, and guidance. The standard should be reviewed in its entirety rather than focusing solely on this summary document.'), ('new requirements for vulnerability assessments', 'The new requirements for vulnerability assessments in PCI DSS version 4.0 include:\\n\\n1. Requirement 4.1.2, which states that there is a new requirement for roles and responsibilities related to vulnerability assessments.\\n2. Requirement 4.2.1, which requires organizations to confirm that certificates used for Point-to-Point Network (PAN) transmissions over open, public networks are valid and not expired or revoked. This bullet is a best practice until March 31, 2025.\\n3. Requirement 10.4.2.1, which requires a targeted risk analysis to determine the frequency of log reviews for all other system components.\\n4. Requirement 10.7.2 and 10.7.3, which require organizations to detect, alert, and address failures of critical security control systems promptly.\\n5. Requirement 11.3.1.1 and 11.3.1.2, which require organizations to manage all applicable vulnerabilities (not ranked as high-risk or critical) and perform internal vulnerability scans via authenticated scanning.\\n6. Requirement A1.2.3, which requires multi-tenant service providers to implement processes or mechanisms for reporting and addressing suspected or confirmed security incidents and vulnerabilities.\\n7. Requirement A3.3.1, which requires multi-tenant service providers to detect, alert, and report failures of automated log review mechanisms and automated code review tools in a timely manner.')]}\n",
+      " What are the new requirements for penetration testing in PCI DSS version 4.0?\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " In PCI DSS version 4.0, the new requirements for penetration testing include defining a penetration testing methodology, covering the entire CDE perimeter and critical systems, testing from both inside and outside the network, testing to validate any segmentation and scope-reduction controls, and performing application-layer and network-layer penetration testing.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 62.654s\n",
+      "Total time used: 330.008 s\n",
+      "Number of tokens generated: 587\n",
+      "Average generation speed: 1.779 tokens/s\n",
+      "CPU times: user 4.72 s, sys: 1.16 s, total: 5.88 s\n",
+      "Wall time: 5min 35s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "os.environ[\"HFTGI_RP\"] = \"1.05\"\n",
+    "!python qa_chain_test.py"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "8a022a7b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading env vars from: /home/inflaton/code/projects/chat-with-orca-2/.env\n",
+      "Running on:  Linux-5.10.102.1-microsoft-standard-WSL2-x86_64-with-glibc2.35\n",
+      "MPS is NOT available\n",
+      "CUDA is  available\n",
+      "MPS not available because the current PyTorch install was not built with MPS enabled.\n",
+      "CUDA is available, we have found  1  GPU(s)\n",
+      "NVIDIA GeForce RTX 4080 Laptop GPU\n",
+      "CUDA version: 12.1\n",
+      "hf_embeddings_device_type: cuda:0\n",
+      "hf_pipeline_device_type: cuda:0\n",
+      "load INSTRUCTOR_Transformer\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n",
+      "max_seq_length  512\n",
+      "Completed in 0.645s\n",
+      "Load index from ./data/faiss_1024_512/ with FAISS\n",
+      "Completed in 0.022s\n",
+      "initializing LLM: hftgi\n",
+      "  hf_pipeline_device_type: cuda:0\n",
+      "     load_quantized_model: None\n",
+      "              torch_dtype: torch.float16\n",
+      "                 n_threds: 4\n",
+      "       repetition_penalty: 1.1\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/pydantic/_internal/_fields.py:149: UserWarning: Field \"model_id\" has conflict with protected namespace \"model_\".\n",
+      "\n",
+      "You may be able to resolve this warning by setting `model_config['protected_namespaces'] = ()`.\n",
+      "  warnings.warn(\n",
+      "initialization complete\n",
+      "Completed in 0.012s\n",
+      "\n",
+      "Question: What's PCI DSS?\n",
+      "{'question': \"What's PCI DSS?\", 'chat_history': []}\n",
+      " PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment data.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 33.113s\n",
+      "\n",
+      "Question: Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?\n",
+      "{'question': 'Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment data.')]}\n",
+      " What are the key differences between PCI DSS versions 3.2.1 and 4.0?\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " The key differences between PCI DSS versions 3.2.1 and 4.0 include changes to the introductory sections, general changes to the requirements, testing procedures, and guidance, and additional changes per requirement for requirements 1-12 and the appendices.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 49.610s\n",
+      "\n",
+      "Question: new requirements for vulnerability assessments\n",
+      "{'question': 'new requirements for vulnerability assessments', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment data.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'The key differences between PCI DSS versions 3.2.1 and 4.0 include changes to the introductory sections, general changes to the requirements, testing procedures, and guidance, and additional changes per requirement for requirements 1-12 and the appendices.')]}\n",
+      " What are the new requirements for vulnerability assessments in PCI DSS version 4.0?\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " There are several new requirements related to vulnerability assessments in PCI DSS version 4.0. Some of them include:\n",
+      "\n",
+      "1. Requirement 4.1.2, which states that there should be clear roles and responsibilities defined for vulnerability management.\n",
+      "2. Requirement 4.2.1, which requires organizations to ensure that certificates used for point-to-point encryption (PAN) transmissions over open, public networks are valid and not expired or revoked. This is considered a best practice until March 31, 2025.\n",
+      "3. Requirement 10.4.2.1, which mandates a targeted risk analysis to determine the frequency of log reviews for all other system components.\n",
+      "4. Requirement 10.7.2 and 10.7.3, which require organizations to detect, alert, and address failures of critical security control systems promptly.\n",
+      "5. Requirement 11.3.1.1 and 11.3.1.2, which require managing all applicable vulnerabilities (not ranked as high-risk or critical) and performing internal vulnerability scans via authenticated scanning.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 155.009s\n",
+      "\n",
+      "Question: more on penetration testing\n",
+      "{'question': 'more on penetration testing', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment data.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'The key differences between PCI DSS versions 3.2.1 and 4.0 include changes to the introductory sections, general changes to the requirements, testing procedures, and guidance, and additional changes per requirement for requirements 1-12 and the appendices.'), ('new requirements for vulnerability assessments', 'There are several new requirements related to vulnerability assessments in PCI DSS version 4.0. Some of them include:\\n\\n1. Requirement 4.1.2, which states that there should be clear roles and responsibilities defined for vulnerability management.\\n2. Requirement 4.2.1, which requires organizations to ensure that certificates used for point-to-point encryption (PAN) transmissions over open, public networks are valid and not expired or revoked. This is considered a best practice until March 31, 2025.\\n3. Requirement 10.4.2.1, which mandates a targeted risk analysis to determine the frequency of log reviews for all other system components.\\n4. Requirement 10.7.2 and 10.7.3, which require organizations to detect, alert, and address failures of critical security control systems promptly.\\n5. Requirement 11.3.1.1 and 11.3.1.2, which require managing all applicable vulnerabilities (not ranked as high-risk or critical) and performing internal vulnerability scans via authenticated scanning.')]}\n",
+      " What are some common methods used in penetration testing, and how do they help organizations meet PCI DSS requirements?\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " Some common methods used in penetration testing include network-layer testing, application-layer testing, and exploiting known vulnerabilities. These methods help organizations meet PCI DSS requirements by identifying and addressing potential security risks and weaknesses in their networks and applications, ensuring compliance with the standard's requirements related to network and application security.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 66.451s\n",
+      "Total time used: 304.184 s\n",
+      "Number of tokens generated: 502\n",
+      "Average generation speed: 1.650 tokens/s\n",
+      "CPU times: user 4.21 s, sys: 1.29 s, total: 5.5 s\n",
+      "Wall time: 5min 8s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "os.environ[\"HFTGI_RP\"] = \"1.10\"\n",
+    "!python qa_chain_test.py"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "db48dcc4",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading env vars from: /home/inflaton/code/projects/chat-with-orca-2/.env\n",
+      "Running on:  Linux-5.10.102.1-microsoft-standard-WSL2-x86_64-with-glibc2.35\n",
+      "MPS is NOT available\n",
+      "CUDA is  available\n",
+      "MPS not available because the current PyTorch install was not built with MPS enabled.\n",
+      "CUDA is available, we have found  1  GPU(s)\n",
+      "NVIDIA GeForce RTX 4080 Laptop GPU\n",
+      "CUDA version: 12.1\n",
+      "hf_embeddings_device_type: cuda:0\n",
+      "hf_pipeline_device_type: cuda:0\n",
+      "load INSTRUCTOR_Transformer\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n",
+      "max_seq_length  512\n",
+      "Completed in 0.623s\n",
+      "Load index from ./data/faiss_1024_512/ with FAISS\n",
+      "Completed in 0.019s\n",
+      "initializing LLM: hftgi\n",
+      "  hf_pipeline_device_type: cuda:0\n",
+      "     load_quantized_model: None\n",
+      "              torch_dtype: torch.float16\n",
+      "                 n_threds: 4\n",
+      "       repetition_penalty: 1.15\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/pydantic/_internal/_fields.py:149: UserWarning: Field \"model_id\" has conflict with protected namespace \"model_\".\n",
+      "\n",
+      "You may be able to resolve this warning by setting `model_config['protected_namespaces'] = ()`.\n",
+      "  warnings.warn(\n",
+      "initialization complete\n",
+      "Completed in 0.013s\n",
+      "\n",
+      "Question: What's PCI DSS?\n",
+      "{'question': \"What's PCI DSS?\", 'chat_history': []}\n",
+      " PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment data.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 33.804s\n",
+      "\n",
+      "Question: Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?\n",
+      "{'question': 'Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment data.')]}\n",
+      " Summarize the differences between PCI DSS versions 3.2.1 and 4.0.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " The main difference between PCI DSS version 3.2.1 and 4.0 is that version 4.0 has updates to align with the changes from PCI DSS v3.2.1 to PCI DSS v4.0.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 47.695s\n",
+      "\n",
+      "Question: new requirements for vulnerability assessments\n",
+      "{'question': 'new requirements for vulnerability assessments', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment data.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'The main difference between PCI DSS version 3.2.1 and 4.0 is that version 4.0 has updates to align with the changes from PCI DSS v3.2.1 to PCI DSS v4.0.')]}\n",
+      " What are the new requirements for vulnerability assessments under PCI DSS version 4.0?\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " There are no specific changes mentioned for vulnerability assessments in the provided text.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 36.644s\n",
+      "\n",
+      "Question: more on penetration testing\n",
+      "{'question': 'more on penetration testing', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment data.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'The main difference between PCI DSS version 3.2.1 and 4.0 is that version 4.0 has updates to align with the changes from PCI DSS v3.2.1 to PCI DSS v4.0.'), ('new requirements for vulnerability assessments', 'There are no specific changes mentioned for vulnerability assessments in the provided text.')]}\n",
+      " What are some best practices for conducting effective penetration tests?\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " Some best practices for conducting effective penetration tests include performing the testing according to the entity's defined methodology, ensuring the test is conducted at least once every 12 months after significant infrastructure or application changes, using a qualified internal resource or external third-party tester who maintains organizational independence, and considering the latest threats and vulnerabilities experienced in the past year. Additionally, it is important to perform testing from both inside and outside the network, validate segmentation and scope reduction controls, and apply application and network-layer testing. Finally, documenting the approach to assessing and addressing risks posed by exploitable vulnerabilities and security weaknesses found during penetration testing and retaining the results and remediation activities for at least 12 months can help improve the effectiveness of these tests.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 85.748s\n",
+      "Total time used: 203.892 s\n",
+      "Number of tokens generated: 339\n",
+      "Average generation speed: 1.663 tokens/s\n",
+      "CPU times: user 2.72 s, sys: 801 ms, total: 3.52 s\n",
+      "Wall time: 3min 28s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "os.environ[\"HFTGI_RP\"] = \"1.15\"\n",
+    "!python qa_chain_test.py"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebook/nvidia-4090-llama-2-13b.ipynb

@@ -0,0 +1,478 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "874cd58e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "workding dir: /home/inflaton/code/chat-with-orca-2\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import sys\n",
+    "from pathlib import Path\n",
+    "\n",
+    "workding_dir = str(Path.cwd().parent)\n",
+    "os.chdir(workding_dir)\n",
+    "sys.path.append(workding_dir)\n",
+    "print(\"workding dir:\", workding_dir)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "19f967ca",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "os.environ[\"LLM_MODEL_TYPE\"] = \"huggingface\"\n",
+    "os.environ[\"HUGGINGFACE_MODEL_NAME_OR_PATH\"] = \"meta-llama/Llama-2-13b-chat-hf\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "a6d96660",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading env vars from: /home/inflaton/code/chat-with-orca-2/.env\n",
+      "Running on:  Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.35\n",
+      "MPS is NOT available\n",
+      "CUDA is  available\n",
+      "MPS not available because the current PyTorch install was not built with MPS enabled.\n",
+      "CUDA is available, we have found  1  GPU(s)\n",
+      "NVIDIA GeForce RTX 4090\n",
+      "CUDA version: 12.1\n",
+      "hf_embeddings_device_type: cuda:0\n",
+      "hf_pipeline_device_type: cuda:0\n",
+      "load INSTRUCTOR_Transformer\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n",
+      "max_seq_length  512\n",
+      "Completed in 0.974s\n",
+      "Load index from ./data/faiss_1024_512/ with FAISS\n",
+      "Completed in 0.024s\n",
+      "initializing LLM: huggingface\n",
+      "  hf_pipeline_device_type: cuda:0\n",
+      "     load_quantized_model: None\n",
+      "              torch_dtype: torch.bfloat16\n",
+      "                 n_threds: 4\n",
+      "            loading model: meta-llama/Llama-2-13b-chat-hf\n",
+      "            HF auth token: None\n",
+      "       repetition_penalty: 1.05\n",
+      "Loading checkpoint shards: 100%|██████████████████| 3/3 [00:08<00:00,  2.78s/it]\n",
+      "initialization complete\n",
+      "Completed in 20.662s\n",
+      "\n",
+      "Question: What's PCI DSS?\n",
+      "{'question': \"What's PCI DSS?\", 'chat_history': []}\n",
+      "PCI DSS is the Payment Card Industry Data Security Standard, a global standard that provides a baseline of technical and operational requirements to protect payment data. It's a standard that helps protect payment account data by providing a baseline of technical and operational requirements designed to ensure the secure handling of cardholder data.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 46.005s\n",
+      "\n",
+      "Question: Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?\n",
+      "{'question': 'Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'chat_history': [(\"What's PCI DSS?\", \"PCI DSS is the Payment Card Industry Data Security Standard, a global standard that provides a baseline of technical and operational requirements to protect payment data. It's a standard that helps protect payment account data by providing a baseline of technical and operational requirements designed to ensure the secure handling of cardholder data.\")]}\n",
+      "What are the key changes between PCI DSS version 3.2.1 and version 4.0?\n",
+      "\n",
+      "Please rephrase the follow-up question to be a standalone question.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " What are the key changes between PCI DSS version 3.2.1 and version 4.0?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 40.791s\n",
+      "\n",
+      "Question: new requirements for vulnerability assessments\n",
+      "{'question': 'new requirements for vulnerability assessments', 'chat_history': [(\"What's PCI DSS?\", \"PCI DSS is the Payment Card Industry Data Security Standard, a global standard that provides a baseline of technical and operational requirements to protect payment data. It's a standard that helps protect payment account data by providing a baseline of technical and operational requirements designed to ensure the secure handling of cardholder data.\"), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'What are the key changes between PCI DSS version 3.2.1 and version 4.0?')]}\n",
+      "What are the new requirements for vulnerability assessments in PCI DSS version 4.0?\n",
+      "\n",
+      "Please provide the rephrased follow-up question as a standalone question in its original language.\n",
+      "\n",
+      "The follow-up question should be:\n",
+      "\n",
+      "What are the new requirements for vulnerability assessments in PCI DSS version 4.0?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "According to the provided document, there are two new requirements for vulnerability assessments in PCI DSS version 4.0:\n",
+      "\n",
+      "1. Maintain an inventory of bespoke and custom software (6.3.2).\n",
+      "2. Manage all other applicable vulnerabilities (those not ranked as high-risk or critical) found during internal vulnerability scans (11.3.1.1). This requirement is a best practice until March 31, 2025.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 100.915s\n",
+      "\n",
+      "Question: more on penetration testing\n",
+      "{'question': 'more on penetration testing', 'chat_history': [(\"What's PCI DSS?\", \"PCI DSS is the Payment Card Industry Data Security Standard, a global standard that provides a baseline of technical and operational requirements to protect payment data. It's a standard that helps protect payment account data by providing a baseline of technical and operational requirements designed to ensure the secure handling of cardholder data.\"), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'What are the key changes between PCI DSS version 3.2.1 and version 4.0?'), ('new requirements for vulnerability assessments', 'According to the provided document, there are two new requirements for vulnerability assessments in PCI DSS version 4.0:\\n\\n1. Maintain an inventory of bespoke and custom software (6.3.2).\\n2. Manage all other applicable vulnerabilities (those not ranked as high-risk or critical) found during internal vulnerability scans (11.3.1.1). This requirement is a best practice until March 31, 2025.')]}\n",
+      "What are the new requirements for penetration testing in PCI DSS version 4.0?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " Based on the provided information, the new requirements for penetration testing in PCI DSS version 4.0 include the following:\n",
+      "\n",
+      "1. Penetration testing must be performed regularly, and exploitable vulnerabilities and security weaknesses must be corrected.\n",
+      "2. The penetration testing methodology must include industry-accepted approaches, coverage for the entire CDE perimeter and critical systems, testing from both inside and outside the network, and testing to validate any segmentation and scope-reduction controls.\n",
+      "3. Application-layer penetration testing must be performed to identify at least the vulnerabilities listed in Requirement 6.2.4.\n",
+      "4. Network-layer penetration tests must encompass all components that support network functions as well as operating systems.\n",
+      "5. The assessor must examine the scope of work and results from the most recent external penetration test to verify that penetration testing is performed according to all elements specified in this requirement.\n",
+      "6. The assessor must interview personnel to verify that the external penetration test was performed by a qualified internal resource or qualified external third-party and that organizational independence of the tester exists.\n",
+      "\n",
+      "Note: These requirements are based on the provided PCI DSS version 4.0 ROC Template r1 and Summary of Changes r2, and may not be comprehensive or up-to-date. It is important to consult the latest version of the PCI DSS documentation for the most current and accurate information.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 181.373s\n",
+      "Total time used: 369.084 s\n",
+      "Number of tokens generated: 677\n",
+      "Average generation speed: 1.834 tokens/s\n",
+      "CPU times: user 3.36 s, sys: 1.15 s, total: 4.51 s\n",
+      "Wall time: 6min 35s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "os.environ[\"HF_RP\"] = \"1.05\"\n",
+    "!python qa_chain_test.py"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "8a022a7b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading env vars from: /home/inflaton/code/chat-with-orca-2/.env\n",
+      "Running on:  Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.35\n",
+      "MPS is NOT available\n",
+      "CUDA is  available\n",
+      "MPS not available because the current PyTorch install was not built with MPS enabled.\n",
+      "CUDA is available, we have found  1  GPU(s)\n",
+      "NVIDIA GeForce RTX 4090\n",
+      "CUDA version: 12.1\n",
+      "hf_embeddings_device_type: cuda:0\n",
+      "hf_pipeline_device_type: cuda:0\n",
+      "load INSTRUCTOR_Transformer\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n",
+      "max_seq_length  512\n",
+      "Completed in 0.941s\n",
+      "Load index from ./data/faiss_1024_512/ with FAISS\n",
+      "Completed in 0.025s\n",
+      "initializing LLM: huggingface\n",
+      "  hf_pipeline_device_type: cuda:0\n",
+      "     load_quantized_model: None\n",
+      "              torch_dtype: torch.bfloat16\n",
+      "                 n_threds: 4\n",
+      "            loading model: meta-llama/Llama-2-13b-chat-hf\n",
+      "            HF auth token: None\n",
+      "       repetition_penalty: 1.1\n",
+      "Loading checkpoint shards: 100%|██████████████████| 3/3 [00:18<00:00,  6.31s/it]\n",
+      "initialization complete\n",
+      "Completed in 37.005s\n",
+      "\n",
+      "Question: What's PCI DSS?\n",
+      "{'question': \"What's PCI DSS?\", 'chat_history': []}\n",
+      "PCI DSS stands for Payment Card Industry Data Security Standard. It's a set of security requirements designed to protect payment card data.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 36.327s\n",
+      "\n",
+      "Question: Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?\n",
+      "{'question': 'Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'chat_history': [(\"What's PCI DSS?\", \"PCI DSS stands for Payment Card Industry Data Security Standard. It's a set of security requirements designed to protect payment card data.\")]}\n",
+      "What are the key changes between PCI DSS version 3.2.1 and version 4.0?\n",
+      "\n",
+      "Please help me with this! Thank you!\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " Based on the provided document, here are the key changes between PCI DSS version 3.2.1 and version 4.0:\n",
+      "\n",
+      "1. Introduction: The introduction has been updated to reflect the new version number (4.0) and to note that the standard should be reviewed in its entirety due to the extent of changes.\n",
+      "\n",
+      "2. Change Types: The document highlights three types of changes: modifications, clarifications, and rephrased text.\n",
+      "\n",
+      "3. Summary of Changes to PCI DSS Introductory Sections: This section summarizes changes made to the introductory sections of the standard, including updates to the definition of \"cardholder data\" and the addition of a new requirement for multi-factor authentication.\n",
+      "\n",
+      "4. Summary of General Changes to PCI DSS Requirements: This section summarizes changes made throughout the requirements, testing procedures, and guidance, including updates to requirements related to password policies, incident response, and vulnerability management.\n",
+      "\n",
+      "5. Additional Changes per Requirement: This section provides a detailed list of changes made to each of the 12 requirements, including updates to requirements related to network segmentation, access control, and security testing.\n",
+      "\n",
+      "Based on this information, it appears that the key changes between PCI DSS version 3.2.1 and version 4.0 include updates to the introductory sections, requirements related to password policies, incident response, and vulnerability management, as well as additional changes to each of the 12 requirements.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 195.181s\n",
+      "\n",
+      "Question: new requirements for vulnerability assessments\n",
+      "{'question': 'new requirements for vulnerability assessments', 'chat_history': [(\"What's PCI DSS?\", \"PCI DSS stands for Payment Card Industry Data Security Standard. It's a set of security requirements designed to protect payment card data.\"), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'Based on the provided document, here are the key changes between PCI DSS version 3.2.1 and version 4.0:\\n\\n1. Introduction: The introduction has been updated to reflect the new version number (4.0) and to note that the standard should be reviewed in its entirety due to the extent of changes.\\n\\n2. Change Types: The document highlights three types of changes: modifications, clarifications, and rephrased text.\\n\\n3. Summary of Changes to PCI DSS Introductory Sections: This section summarizes changes made to the introductory sections of the standard, including updates to the definition of \"cardholder data\" and the addition of a new requirement for multi-factor authentication.\\n\\n4. Summary of General Changes to PCI DSS Requirements: This section summarizes changes made throughout the requirements, testing procedures, and guidance, including updates to requirements related to password policies, incident response, and vulnerability management.\\n\\n5. Additional Changes per Requirement: This section provides a detailed list of changes made to each of the 12 requirements, including updates to requirements related to network segmentation, access control, and security testing.\\n\\nBased on this information, it appears that the key changes between PCI DSS version 3.2.1 and version 4.0 include updates to the introductory sections, requirements related to password policies, incident response, and vulnerability management, as well as additional changes to each of the 12 requirements.')]}\n",
+      "What are the new requirements for vulnerability assessments in PCI DSS version 4.0?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " According to the provided document, there are several new requirements for vulnerability assessments in PCI DSS version 4.0. These include:\n",
+      "\n",
+      "* Performing a targeted risk analysis to determine the frequency of log reviews for all other system components (10.4.2.1)\n",
+      "* Detecting, alerting, and addressing promptly failures of critical security control systems (10.7.2 and 10.7.3)\n",
+      "* Documenting, assigning, and understanding roles and responsibilities for performing activities in Requirement 11 (11.1.2)\n",
+      "* Managing all other applicable vulnerabilities (those not ranked as high-risk or critical) (11.3.1.1)\n",
+      "* Performing internal vulnerability scans via authenticated scanning (11.3.1.2)\n",
+      "* Supporting multi-tenant service providers (11.4.7)\n",
+      "* Implementing processes or mechanisms for reporting and addressing suspected or confirmed security incidents and vulnerabilities (A1.2.3)\n",
+      "* Detecting, alerting, and reporting in a timely manner failures of automated log review mechanisms and automated code review tools (A3.3.1)\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 170.772s\n",
+      "\n",
+      "Question: more on penetration testing\n",
+      "{'question': 'more on penetration testing', 'chat_history': [(\"What's PCI DSS?\", \"PCI DSS stands for Payment Card Industry Data Security Standard. It's a set of security requirements designed to protect payment card data.\"), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'Based on the provided document, here are the key changes between PCI DSS version 3.2.1 and version 4.0:\\n\\n1. Introduction: The introduction has been updated to reflect the new version number (4.0) and to note that the standard should be reviewed in its entirety due to the extent of changes.\\n\\n2. Change Types: The document highlights three types of changes: modifications, clarifications, and rephrased text.\\n\\n3. Summary of Changes to PCI DSS Introductory Sections: This section summarizes changes made to the introductory sections of the standard, including updates to the definition of \"cardholder data\" and the addition of a new requirement for multi-factor authentication.\\n\\n4. Summary of General Changes to PCI DSS Requirements: This section summarizes changes made throughout the requirements, testing procedures, and guidance, including updates to requirements related to password policies, incident response, and vulnerability management.\\n\\n5. Additional Changes per Requirement: This section provides a detailed list of changes made to each of the 12 requirements, including updates to requirements related to network segmentation, access control, and security testing.\\n\\nBased on this information, it appears that the key changes between PCI DSS version 3.2.1 and version 4.0 include updates to the introductory sections, requirements related to password policies, incident response, and vulnerability management, as well as additional changes to each of the 12 requirements.'), ('new requirements for vulnerability assessments', 'According to the provided document, there are several new requirements for vulnerability assessments in PCI DSS version 4.0. These include:\\n\\n* Performing a targeted risk analysis to determine the frequency of log reviews for all other system components (10.4.2.1)\\n* Detecting, alerting, and addressing promptly failures of critical security control systems (10.7.2 and 10.7.3)\\n* Documenting, assigning, and understanding roles and responsibilities for performing activities in Requirement 11 (11.1.2)\\n* Managing all other applicable vulnerabilities (those not ranked as high-risk or critical) (11.3.1.1)\\n* Performing internal vulnerability scans via authenticated scanning (11.3.1.2)\\n* Supporting multi-tenant service providers (11.4.7)\\n* Implementing processes or mechanisms for reporting and addressing suspected or confirmed security incidents and vulnerabilities (A1.2.3)\\n* Detecting, alerting, and reporting in a timely manner failures of automated log review mechanisms and automated code review tools (A3.3.1)')]}\n",
+      "What are the new requirements for penetration testing in PCI DSS version 4.0?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " Based on the provided information, there are two new requirements for penetration testing in PCI DSS version 4.0. Firstly, the penetration testing methodology must include industry-accepted penetration testing approaches, coverage for the entire CDE perimeter, and testing from both inside and outside the network. Secondly, the assessment must examine the scope of work and results from the most recent external penetration test to ensure that penetration testing is performed according to all elements specified in this requirement. Additionally, the assessor must interview personnel to verify that the external penetration test was performed by a qualified internal resource or qualified external third-party and that organizational independence of the tester exists.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 103.536s\n",
+      "Total time used: 505.816 s\n",
+      "Number of tokens generated: 881\n",
+      "Average generation speed: 1.742 tokens/s\n",
+      "CPU times: user 4.76 s, sys: 1.45 s, total: 6.21 s\n",
+      "Wall time: 9min 9s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "os.environ[\"HF_RP\"] = \"1.10\"\n",
+    "!python qa_chain_test.py"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "db48dcc4",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading env vars from: /home/inflaton/code/chat-with-orca-2/.env\n",
+      "Running on:  Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.35\n",
+      "MPS is NOT available\n",
+      "CUDA is  available\n",
+      "MPS not available because the current PyTorch install was not built with MPS enabled.\n",
+      "CUDA is available, we have found  1  GPU(s)\n",
+      "NVIDIA GeForce RTX 4090\n",
+      "CUDA version: 12.1\n",
+      "hf_embeddings_device_type: cuda:0\n",
+      "hf_pipeline_device_type: cuda:0\n",
+      "load INSTRUCTOR_Transformer\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n",
+      "max_seq_length  512\n",
+      "Completed in 1.202s\n",
+      "Load index from ./data/faiss_1024_512/ with FAISS\n",
+      "Completed in 0.040s\n",
+      "initializing LLM: huggingface\n",
+      "  hf_pipeline_device_type: cuda:0\n",
+      "     load_quantized_model: None\n",
+      "              torch_dtype: torch.bfloat16\n",
+      "                 n_threds: 4\n",
+      "            loading model: meta-llama/Llama-2-13b-chat-hf\n",
+      "            HF auth token: None\n",
+      "       repetition_penalty: 1.15\n",
+      "Loading checkpoint shards: 100%|██████████████████| 3/3 [00:08<00:00,  2.72s/it]\n",
+      "initialization complete\n",
+      "Completed in 20.624s\n",
+      "\n",
+      "Question: What's PCI DSS?\n",
+      "{'question': \"What's PCI DSS?\", 'chat_history': []}\n",
+      "PCI DSS stands for Payment Card Industry Data Security Standard. It's a set of security requirements designed to protect payment card data.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 31.125s\n",
+      "\n",
+      "Question: Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?\n",
+      "{'question': 'Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'chat_history': [(\"What's PCI DSS?\", \"PCI DSS stands for Payment Card Industry Data Security Standard. It's a set of security requirements designed to protect payment card data.\")]}\n",
+      "What are the key changes between PCI DSS version 3.2.1 and version 4.0?\n",
+      "\n",
+      "Please help me with this! Thank you so much!\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " Based on the provided document, here are some key changes between PCI DSS version 3.2.1 and version 4.0:\n",
+      "\n",
+      "1. The introduction has been updated to reflect the new version number (4.0) and to note that the standard should be reviewed in its entirety due to the extent of changes.\n",
+      "2. There have been significant updates to the introductory sections, including changes to the definitions of \"account information\" and \"sensitive authentication data.\"\n",
+      "3. Requirement 8.3.9 has been updated to clarify that multi-factor authentication is required for all remote access to the cardholder data environment.\n",
+      "4. Requirement 6.3.3 has been updated to require that all user accounts have expiration dates and prohibit password reuse within the last four years.\n",
+      "5. A new requirement (3.6.1.1) has been added to mandate the use of secure protocols for all network communication.\n",
+      "6. Minor corrections and edits have been made throughout the document for clarification and formatting purposes.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 139.938s\n",
+      "\n",
+      "Question: new requirements for vulnerability assessments\n",
+      "{'question': 'new requirements for vulnerability assessments', 'chat_history': [(\"What's PCI DSS?\", \"PCI DSS stands for Payment Card Industry Data Security Standard. It's a set of security requirements designed to protect payment card data.\"), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'Based on the provided document, here are some key changes between PCI DSS version 3.2.1 and version 4.0:\\n\\n1. The introduction has been updated to reflect the new version number (4.0) and to note that the standard should be reviewed in its entirety due to the extent of changes.\\n2. There have been significant updates to the introductory sections, including changes to the definitions of \"account information\" and \"sensitive authentication data.\"\\n3. Requirement 8.3.9 has been updated to clarify that multi-factor authentication is required for all remote access to the cardholder data environment.\\n4. Requirement 6.3.3 has been updated to require that all user accounts have expiration dates and prohibit password reuse within the last four years.\\n5. A new requirement (3.6.1.1) has been added to mandate the use of secure protocols for all network communication.\\n6. Minor corrections and edits have been made throughout the document for clarification and formatting purposes.')]}\n",
+      "What are the new requirements for vulnerability assessments in PCI DSS version 4.0?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " According to the provided document, there are several new requirements for vulnerability assessments in PCI DSS version 4.0. These include:\n",
+      "\n",
+      "* Performing a targeted risk analysis to determine the frequency of log reviews for all other system components (10.4.2.1)\n",
+      "* Detecting, alerting, and addressing failures of critical security control systems promptly (10.7.2 and 10.7.3)\n",
+      "* Documenting, assigning, and understanding roles and responsibilities for performing activities in Requirement 11 (11.1.2)\n",
+      "* Managing all other applicable vulnerabilities (not ranked as high-risk or critical) (11.3.1.1)\n",
+      "* Performing internal vulnerability scans via authenticated scanning (11.3.1.2)\n",
+      "* Supporting multi-tenant services (11.4.7)\n",
+      "* Implementing processes or mechanisms for reporting and addressing suspected or confirmed security incidents and vulnerabilities (A1.2.3)\n",
+      "* Detecting, alerting, and reporting failures of automated log review mechanisms and automated code review tools in a timely manner (A3.3.1)\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 164.073s\n",
+      "\n",
+      "Question: more on penetration testing\n",
+      "{'question': 'more on penetration testing', 'chat_history': [(\"What's PCI DSS?\", \"PCI DSS stands for Payment Card Industry Data Security Standard. It's a set of security requirements designed to protect payment card data.\"), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'Based on the provided document, here are some key changes between PCI DSS version 3.2.1 and version 4.0:\\n\\n1. The introduction has been updated to reflect the new version number (4.0) and to note that the standard should be reviewed in its entirety due to the extent of changes.\\n2. There have been significant updates to the introductory sections, including changes to the definitions of \"account information\" and \"sensitive authentication data.\"\\n3. Requirement 8.3.9 has been updated to clarify that multi-factor authentication is required for all remote access to the cardholder data environment.\\n4. Requirement 6.3.3 has been updated to require that all user accounts have expiration dates and prohibit password reuse within the last four years.\\n5. A new requirement (3.6.1.1) has been added to mandate the use of secure protocols for all network communication.\\n6. Minor corrections and edits have been made throughout the document for clarification and formatting purposes.'), ('new requirements for vulnerability assessments', 'According to the provided document, there are several new requirements for vulnerability assessments in PCI DSS version 4.0. These include:\\n\\n* Performing a targeted risk analysis to determine the frequency of log reviews for all other system components (10.4.2.1)\\n* Detecting, alerting, and addressing failures of critical security control systems promptly (10.7.2 and 10.7.3)\\n* Documenting, assigning, and understanding roles and responsibilities for performing activities in Requirement 11 (11.1.2)\\n* Managing all other applicable vulnerabilities (not ranked as high-risk or critical) (11.3.1.1)\\n* Performing internal vulnerability scans via authenticated scanning (11.3.1.2)\\n* Supporting multi-tenant services (11.4.7)\\n* Implementing processes or mechanisms for reporting and addressing suspected or confirmed security incidents and vulnerabilities (A1.2.3)\\n* Detecting, alerting, and reporting failures of automated log review mechanisms and automated code review tools in a timely manner (A3.3.1)')]}\n",
+      "What are the new requirements for penetration testing in PCI DSS version 4.0?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " Based on the provided information, there are two new requirements for penetration testing in PCI DSS version 4.0. Firstly, the penetration testing methodology must include industry-accepted penetration testing approaches, coverage for the entire CDE perimeter, and testing from both inside and outside the network. Secondly, the assessment must examine the scope of work and results from the most recent external penetration test to ensure that penetration testing is performed according to all elements specified in this requirement. Additionally, the assessor must interview personnel to verify that the external penetration test was performed by a qualified internal resource or qualified external third-party and that organizational independence of the tester exists.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 100.293s\n",
+      "Total time used: 435.429 s\n",
+      "Number of tokens generated: 777\n",
+      "Average generation speed: 1.784 tokens/s\n",
+      "CPU times: user 3.96 s, sys: 1.08 s, total: 5.04 s\n",
+      "Wall time: 7min 44s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "os.environ[\"HF_RP\"] = \"1.15\"\n",
+    "!python qa_chain_test.py"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebook/nvidia-4090-llama-2-7b.ipynb

@@ -0,0 +1,451 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "874cd58e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "workding dir: /home/inflaton/code/chat-with-orca-2\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import sys\n",
+    "from pathlib import Path\n",
+    "\n",
+    "workding_dir = str(Path.cwd().parent)\n",
+    "os.chdir(workding_dir)\n",
+    "sys.path.append(workding_dir)\n",
+    "print(\"workding dir:\", workding_dir)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "19f967ca",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "os.environ[\"LLM_MODEL_TYPE\"] = \"huggingface\"\n",
+    "os.environ[\"HUGGINGFACE_MODEL_NAME_OR_PATH\"] = \"meta-llama/Llama-2-7b-chat-hf\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "a6d96660",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading env vars from: /home/inflaton/code/chat-with-orca-2/.env\n",
+      "Running on:  Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.35\n",
+      "MPS is NOT available\n",
+      "CUDA is  available\n",
+      "MPS not available because the current PyTorch install was not built with MPS enabled.\n",
+      "CUDA is available, we have found  1  GPU(s)\n",
+      "NVIDIA GeForce RTX 4090\n",
+      "CUDA version: 12.1\n",
+      "hf_embeddings_device_type: cuda:0\n",
+      "hf_pipeline_device_type: cuda:0\n",
+      "load INSTRUCTOR_Transformer\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n",
+      "max_seq_length  512\n",
+      "Completed in 0.947s\n",
+      "Load index from ./data/faiss_1024_512/ with FAISS\n",
+      "Completed in 0.028s\n",
+      "initializing LLM: huggingface\n",
+      "  hf_pipeline_device_type: cuda:0\n",
+      "     load_quantized_model: None\n",
+      "              torch_dtype: torch.bfloat16\n",
+      "                 n_threds: 4\n",
+      "            loading model: meta-llama/Llama-2-7b-chat-hf\n",
+      "            HF auth token: None\n",
+      "       repetition_penalty: 1.05\n",
+      "Loading checkpoint shards: 100%|██████████████████| 2/2 [00:07<00:00,  3.74s/it]\n",
+      "initialization complete\n",
+      "Completed in 42.539s\n",
+      "\n",
+      "Question: What's PCI DSS?\n",
+      "{'question': \"What's PCI DSS?\", 'chat_history': []}\n",
+      "PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment account data. The standard was developed by the PCI Security Standards Council and is widely adopted by organizations that handle payment card data. PCI DSS v4.0 is the latest version of the standard, which includes new controls to address sophisticated cyber attacks.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 4.870s\n",
+      "\n",
+      "Question: Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?\n",
+      "{'question': 'Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment account data. The standard was developed by the PCI Security Standards Council and is widely adopted by organizations that handle payment card data. PCI DSS v4.0 is the latest version of the standard, which includes new controls to address sophisticated cyber attacks.')]}\n",
+      "What are the key changes made in PCI DSS version 4.0 compared to version 3.2.1?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " Based on the provided document, the key changes made in PCI DSS version 4.0 compared to version 3.2.1 include:\n",
+      "\n",
+      "* Update of principal requirement title to reflect the focus on \"network security controls\"\n",
+      "* Replacement of \"firewalls\" and \"routers\" with \"network security controls\" to support a broader range of technologies used to meet the security objectives traditionally met by firewalls\n",
+      "* Removal of requirement for \"Description of groups, roles, and responsibilities for management of network components\" and replacement with a general requirement for roles and responsibilities for Requirement 1\n",
+      "* Refocusing of former \"null\" requirement (all content pointed to other requirements) on defining,\n",
+      "\n",
+      "Unhelpful Answer:  I'm not sure, can you provide more context or information about the changes made in PCI DSS version 4.0 compared to version 3.2.1?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 5.714s\n",
+      "\n",
+      "Question: new requirements for vulnerability assessments\n",
+      "{'question': 'new requirements for vulnerability assessments', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment account data. The standard was developed by the PCI Security Standards Council and is widely adopted by organizations that handle payment card data. PCI DSS v4.0 is the latest version of the standard, which includes new controls to address sophisticated cyber attacks.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'Based on the provided document, the key changes made in PCI DSS version 4.0 compared to version 3.2.1 include:\\n\\n* Update of principal requirement title to reflect the focus on \"network security controls\"\\n* Replacement of \"firewalls\" and \"routers\" with \"network security controls\" to support a broader range of technologies used to meet the security objectives traditionally met by firewalls\\n* Removal of requirement for \"Description of groups, roles, and responsibilities for management of network components\" and replacement with a general requirement for roles and responsibilities for Requirement 1\\n* Refocusing of former \"null\" requirement (all content pointed to other requirements) on defining,\\n\\nUnhelpful Answer: I\\'m not sure, can you provide more context or information about the changes made in PCI DSS version 4.0 compared to version 3.2.1?')]}\n",
+      "What are the new requirements for vulnerability assessments in PCI DSS version 4.0 compared to version 3.2.1?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " According to the provided summary of changes, there are two new requirements for vulnerability assessments in PCI DSS version 4.0 compared to version 3.2.1:\n",
+      "\n",
+      "* Requirement 4.1.2: New requirement for roles and responsibilities.\n",
+      "* Requirement 6.3.2: New requirement to maintain an inventory of bespoke and custom software.\n",
+      "\n",
+      "Unhelpful Answer:  I don't know the answer to your question.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 3.566s\n",
+      "\n",
+      "Question: more on penetration testing\n",
+      "{'question': 'more on penetration testing', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment account data. The standard was developed by the PCI Security Standards Council and is widely adopted by organizations that handle payment card data. PCI DSS v4.0 is the latest version of the standard, which includes new controls to address sophisticated cyber attacks.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'Based on the provided document, the key changes made in PCI DSS version 4.0 compared to version 3.2.1 include:\\n\\n* Update of principal requirement title to reflect the focus on \"network security controls\"\\n* Replacement of \"firewalls\" and \"routers\" with \"network security controls\" to support a broader range of technologies used to meet the security objectives traditionally met by firewalls\\n* Removal of requirement for \"Description of groups, roles, and responsibilities for management of network components\" and replacement with a general requirement for roles and responsibilities for Requirement 1\\n* Refocusing of former \"null\" requirement (all content pointed to other requirements) on defining,\\n\\nUnhelpful Answer: I\\'m not sure, can you provide more context or information about the changes made in PCI DSS version 4.0 compared to version 3.2.1?'), ('new requirements for vulnerability assessments', \"According to the provided summary of changes, there are two new requirements for vulnerability assessments in PCI DSS version 4.0 compared to version 3.2.1:\\n\\n* Requirement 4.1.2: New requirement for roles and responsibilities.\\n* Requirement 6.3.2: New requirement to maintain an inventory of bespoke and custom software.\\n\\nUnhelpful Answer: I don't know the answer to your question.\")]}\n",
+      "What are the new requirements for penetration testing in PCI DSS version 4.0 compared to version 3.2.1?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " The new requirement for penetration testing in PCI DSS version 4.0 is 11.4.1, which states that external and internal penetration testing is regularly performed, and exploitable vulnerabilities and security weaknesses are corrected. This requirement includes a detailed list of what the penetration testing methodology should cover, such as industry-accepted penetration testing approaches, coverage of the entire CDE perimeter and critical systems, testing from both inside and outside the network, and application-layer penetration testing to identify vulnerabilities listed in Requirement 6.2.4. Additionally, the requirement includes network-layer penetration tests that encompass all components that support network functions as well as operating systems.\n",
+      "\n",
+      "Unhelpful Answer:  I don't know the answer to your question.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 5.318s\n",
+      "Total time used: 19.468 s\n",
+      "Number of tokens generated: 679\n",
+      "Average generation speed: 34.878 tokens/s\n",
+      "CPU times: user 638 ms, sys: 275 ms, total: 913 ms\n",
+      "Wall time: 1min 7s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "os.environ[\"HF_RP\"] = \"1.05\"\n",
+    "!python qa_chain_test.py"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "8a022a7b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading env vars from: /home/inflaton/code/chat-with-orca-2/.env\n",
+      "Running on:  Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.35\n",
+      "MPS is NOT available\n",
+      "CUDA is  available\n",
+      "MPS not available because the current PyTorch install was not built with MPS enabled.\n",
+      "CUDA is available, we have found  1  GPU(s)\n",
+      "NVIDIA GeForce RTX 4090\n",
+      "CUDA version: 12.1\n",
+      "hf_embeddings_device_type: cuda:0\n",
+      "hf_pipeline_device_type: cuda:0\n",
+      "load INSTRUCTOR_Transformer\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n",
+      "max_seq_length  512\n",
+      "Completed in 0.498s\n",
+      "Load index from ./data/faiss_1024_512/ with FAISS\n",
+      "Completed in 0.025s\n",
+      "initializing LLM: huggingface\n",
+      "  hf_pipeline_device_type: cuda:0\n",
+      "     load_quantized_model: None\n",
+      "              torch_dtype: torch.bfloat16\n",
+      "                 n_threds: 4\n",
+      "            loading model: meta-llama/Llama-2-7b-chat-hf\n",
+      "            HF auth token: None\n",
+      "       repetition_penalty: 1.1\n",
+      "Loading checkpoint shards: 100%|██████████████████| 2/2 [00:02<00:00,  1.04s/it]\n",
+      "initialization complete\n",
+      "Completed in 7.318s\n",
+      "\n",
+      "Question: What's PCI DSS?\n",
+      "{'question': \"What's PCI DSS?\", 'chat_history': []}\n",
+      "PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment account data. The standard is developed and maintained by the PCI Security Standards Council, and it includes requirements for building and maintaining a secure network and systems, protecting account data, maintaining a vulnerability management program, implementing strong access control measures, and more. The latest version of PCI DSS, version 4.0, was released in December 2022, and it includes new controls to address sophisticated cyber attacks.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 6.128s\n",
+      "\n",
+      "Question: Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?\n",
+      "{'question': 'Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment account data. The standard is developed and maintained by the PCI Security Standards Council, and it includes requirements for building and maintaining a secure network and systems, protecting account data, maintaining a vulnerability management program, implementing strong access control measures, and more. The latest version of PCI DSS, version 4.0, was released in December 2022, and it includes new controls to address sophisticated cyber attacks.')]}\n",
+      "What are the key changes made in PCI DSS version 4.0 compared to version 3.2.1?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "The key changes made in PCI DSS version 4.0 compared to version 3.2.1 include updated principal requirement titles, replaced \"firewalls\" and \"routers\" with \"network security controls\", and refocused former \"null\" requirements on defining roles and responsibilities. Additionally, there were errata updates made in May 2022 to correct the change description for PCI DSS v4.0 requirement 8.3.9 and in December 2022 to add a description of the change made to Requirement 6.3.3 and to correct the entry in the Summary of New Requirements table for Requirement 3.6.1.1.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 4.480s\n",
+      "\n",
+      "Question: new requirements for vulnerability assessments\n",
+      "{'question': 'new requirements for vulnerability assessments', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment account data. The standard is developed and maintained by the PCI Security Standards Council, and it includes requirements for building and maintaining a secure network and systems, protecting account data, maintaining a vulnerability management program, implementing strong access control measures, and more. The latest version of PCI DSS, version 4.0, was released in December 2022, and it includes new controls to address sophisticated cyber attacks.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'The key changes made in PCI DSS version 4.0 compared to version 3.2.1 include updated principal requirement titles, replaced \"firewalls\" and \"routers\" with \"network security controls\", and refocused former \"null\" requirements on defining roles and responsibilities. Additionally, there were errata updates made in May 2022 to correct the change description for PCI DSS v4.0 requirement 8.3.9 and in December 2022 to add a description of the change made to Requirement 6.3.3 and to correct the entry in the Summary of New Requirements table for Requirement 3.6.1.1.')]}\n",
+      "What are the new requirements for vulnerability assessments in PCI DSS version 4.0 compared to previous versions?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "The new requirement for vulnerability assessments in PCI DSS version 4.0 is 6.3.2, which adds a bullet to clarify applicability to vulnerabilities for bespoke and custom and third-party software. Additionally, there are general changes implemented throughout PCI DSS requirements, such as reformatted overview sections, updated guidance, and renumbered requirements and testing procedures. However, I cannot provide an exact answer without knowing the specific context of your environment and the version of PCI DSS you are referring to.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 3.577s\n",
+      "\n",
+      "Question: more on penetration testing\n",
+      "{'question': 'more on penetration testing', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment account data. The standard is developed and maintained by the PCI Security Standards Council, and it includes requirements for building and maintaining a secure network and systems, protecting account data, maintaining a vulnerability management program, implementing strong access control measures, and more. The latest version of PCI DSS, version 4.0, was released in December 2022, and it includes new controls to address sophisticated cyber attacks.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'The key changes made in PCI DSS version 4.0 compared to version 3.2.1 include updated principal requirement titles, replaced \"firewalls\" and \"routers\" with \"network security controls\", and refocused former \"null\" requirements on defining roles and responsibilities. Additionally, there were errata updates made in May 2022 to correct the change description for PCI DSS v4.0 requirement 8.3.9 and in December 2022 to add a description of the change made to Requirement 6.3.3 and to correct the entry in the Summary of New Requirements table for Requirement 3.6.1.1.'), ('new requirements for vulnerability assessments', 'The new requirement for vulnerability assessments in PCI DSS version 4.0 is 6.3.2, which adds a bullet to clarify applicability to vulnerabilities for bespoke and custom and third-party software. Additionally, there are general changes implemented throughout PCI DSS requirements, such as reformatted overview sections, updated guidance, and renumbered requirements and testing procedures. However, I cannot provide an exact answer without knowing the specific context of your environment and the version of PCI DSS you are referring to.')]}\n",
+      "What are the new requirements for penetration testing in PCI DSS version 4.0?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "The new requirements for penetration testing in PCI DSS version 4.0 can be found on pages 372-377 of the PCI DSS v4.0 ROC Template r1 published in December 2022. Specifically, the new requirements include:\n",
+      "\n",
+      "* A penetration testing methodology must be defined, documented, and implemented by the entity (Requirement 11.4.1).\n",
+      "* Penetration testing must be conducted regularly and must cover the entire CDE perimeter and critical systems (Requirement 11.4.2).\n",
+      "* Testing must be performed from both inside and outside the network (Requirement 11.4.2).\n",
+      "* Testing must validate any segmentation and scope-reduction controls (Requirement 11.4.2).\n",
+      "* Application-layer penetration testing must be conducted to identify, at a minimum, the vulnerabilities listed in Requirement 6.2.4 (Requirement 11.4.1).\n",
+      "* Network-layer penetration tests must encompass all components that support network functions as well as operating systems (Requirement 11.4.1).\n",
+      "\n",
+      "If you don't know the answer, please say so.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 7.484s\n",
+      "Total time used: 21.670 s\n",
+      "Number of tokens generated: 759\n",
+      "Average generation speed: 35.026 tokens/s\n",
+      "CPU times: user 356 ms, sys: 147 ms, total: 503 ms\n",
+      "Wall time: 33.6 s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "os.environ[\"HF_RP\"] = \"1.10\"\n",
+    "!python qa_chain_test.py"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "db48dcc4",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading env vars from: /home/inflaton/code/chat-with-orca-2/.env\n",
+      "Running on:  Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.35\n",
+      "MPS is NOT available\n",
+      "CUDA is  available\n",
+      "MPS not available because the current PyTorch install was not built with MPS enabled.\n",
+      "CUDA is available, we have found  1  GPU(s)\n",
+      "NVIDIA GeForce RTX 4090\n",
+      "CUDA version: 12.1\n",
+      "hf_embeddings_device_type: cuda:0\n",
+      "hf_pipeline_device_type: cuda:0\n",
+      "load INSTRUCTOR_Transformer\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n",
+      "max_seq_length  512\n",
+      "Completed in 0.454s\n",
+      "Load index from ./data/faiss_1024_512/ with FAISS\n",
+      "Completed in 0.014s\n",
+      "initializing LLM: huggingface\n",
+      "  hf_pipeline_device_type: cuda:0\n",
+      "     load_quantized_model: None\n",
+      "              torch_dtype: torch.bfloat16\n",
+      "                 n_threds: 4\n",
+      "            loading model: meta-llama/Llama-2-7b-chat-hf\n",
+      "            HF auth token: None\n",
+      "       repetition_penalty: 1.15\n",
+      "Loading checkpoint shards: 100%|██████████████████| 2/2 [00:01<00:00,  1.25it/s]\n",
+      "initialization complete\n",
+      "Completed in 5.131s\n",
+      "\n",
+      "Question: What's PCI DSS?\n",
+      "{'question': \"What's PCI DSS?\", 'chat_history': []}\n",
+      "According to the given quick reference guide, PCI DSS stands for \"Payment Card Industry Data Security Standard.\" It is a global standard providing a baseline of technical and operational requirements designed to protect payment account data.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 3.775s\n",
+      "\n",
+      "Question: Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?\n",
+      "{'question': 'Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'chat_history': [(\"What's PCI DSS?\", 'According to the given quick reference guide, PCI DSS stands for \"Payment Card Industry Data Security Standard.\" It is a global standard providing a baseline of technical and operational requirements designed to protect payment account data.')]}\n",
+      "What are the key changes between PCI DSS versions 3.2.1 and 4.0?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "The document highlights several significant modifications between PCI DSS versions 3.2.1 and 4.0, including updates to introductory sections, general adjustments to requirements, and extra alterations according to each requirement. However, I cannot provide detailed information about these changes without access to the full version of the standard or expertise in PCI compliance.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 2.568s\n",
+      "\n",
+      "Question: new requirements for vulnerability assessments\n",
+      "{'question': 'new requirements for vulnerability assessments', 'chat_history': [(\"What's PCI DSS?\", 'According to the given quick reference guide, PCI DSS stands for \"Payment Card Industry Data Security Standard.\" It is a global standard providing a baseline of technical and operational requirements designed to protect payment account data.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'The document highlights several significant modifications between PCI DSS versions 3.2.1 and 4.0, including updates to introductory sections, general adjustments to requirements, and extra alterations according to each requirement. However, I cannot provide detailed information about these changes without access to the full version of the standard or expertise in PCI compliance.')]}\n",
+      "What are some of the new requirements for vulnerability assessments introduced in PCI DSS version 4.0 compared to previous versions?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "According to the provided documents, one of the new requirements for vulnerability assessments introduced in PCI DSS version 4.0 is the addition of a bullet point in requirement 6.3.1 to clarify applicability to vulnerabilities for bespoke and custom and third-party software (see page 10). Additionally, there is a new requirement for maintaining an inventory of trusted keys and certificates (see page 1), which is a best practice until March 31, 2025.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 3.544s\n",
+      "\n",
+      "Question: more on penetration testing\n",
+      "{'question': 'more on penetration testing', 'chat_history': [(\"What's PCI DSS?\", 'According to the given quick reference guide, PCI DSS stands for \"Payment Card Industry Data Security Standard.\" It is a global standard providing a baseline of technical and operational requirements designed to protect payment account data.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'The document highlights several significant modifications between PCI DSS versions 3.2.1 and 4.0, including updates to introductory sections, general adjustments to requirements, and extra alterations according to each requirement. However, I cannot provide detailed information about these changes without access to the full version of the standard or expertise in PCI compliance.'), ('new requirements for vulnerability assessments', 'According to the provided documents, one of the new requirements for vulnerability assessments introduced in PCI DSS version 4.0 is the addition of a bullet point in requirement 6.3.1 to clarify applicability to vulnerabilities for bespoke and custom and third-party software (see page 10). Additionally, there is a new requirement for maintaining an inventory of trusted keys and certificates (see page 1), which is a best practice until March 31, 2025.')]}\n",
+      "Could you explain what penetration testing entails in the context of PCI DSS compliance?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Sure! Penetration testing is a crucial component of PCI DSS compliance, which involves simulating cyber attacks against an organization's computer systems and networks to identify vulnerabilities and weaknesses. The goal of penetration testing is to help organizations strengthen their defenses against attackers who might exploit these vulnerabilities to gain unauthorized access to sensitive data.\n",
+      "\n",
+      "The PCI DSS requires organizations to conduct regular penetration testing to ensure that their systems and networks are secure and protected against potential threats. This includes testing the perimeter of the cardholder data environment (CDE), critical systems, and application layers to identify any vulnerabilities that could be exploited by attackers.\n",
+      "\n",
+      "There are several key aspects of penetration testing that organizations should be aware of when implementing PCI DSS compliance:\n",
+      "\n",
+      "1. Defining a penetration testing methodology: Organizations must define a comprehensive approach to penetration testing that includes industry- accepted techniques and best practices.\n",
+      "2. Coverage of the entire CDE perimeter and critical systems: Penetration testing should cover the entire perimeter of the CDE, including firewalls, routers, switches, and other network devices, as well as critical systems such as databases, servers, and applications.\n",
+      "3. Testing from both inside and outside the network: Penetration testing should involve testing from both inside and outside the network to simulate real-world attacks and identify vulnerabilities that may not be visible from within the network.\n",
+      "4. Validating segmentation and scope-reduction controls: Penetration testing should also validate the effectiveness of segmentation and scope-reduction controls to prevent unauthorized access to sensitive data.\n",
+      "5. Identifying vulnerabilities in application layers: Penetration testing should include testing of application layers to identify any vulnerabilities that could be exploited by attackers.\n",
+      "6. Correction of identified vulnerabilities: Once vulnerabilities are identified through penetration testing, they must be corrected promptly to minimize the risk of attack.\n",
+      "\n",
+      "By understanding the importance of penetration testing in PCI DSS compliance, organizations can take proactive steps to protect their systems and networks from cyber threats and maintain the trust of their customers and stakeholders.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 12.717s\n",
+      "Total time used: 22.604 s\n",
+      "Number of tokens generated: 803\n",
+      "Average generation speed: 35.524 tokens/s\n",
+      "CPU times: user 397 ms, sys: 149 ms, total: 546 ms\n",
+      "Wall time: 31.6 s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "os.environ[\"HF_RP\"] = \"1.15\"\n",
+    "!python qa_chain_test.py"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebook/nvidia-4090-openai.ipynb

@@ -0,0 +1,369 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "874cd58e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "workding dir: /home/inflaton/code/chat-with-orca-2\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import sys\n",
+    "from pathlib import Path\n",
+    "\n",
+    "workding_dir = str(Path.cwd().parent)\n",
+    "os.chdir(workding_dir)\n",
+    "sys.path.append(workding_dir)\n",
+    "print(\"workding dir:\", workding_dir)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "19f967ca",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "os.environ[\"LLM_MODEL_TYPE\"] = \"openai\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "a6d96660",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading env vars from: /home/inflaton/code/chat-with-orca-2/.env\n",
+      "Running on:  Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.35\n",
+      "MPS is NOT available\n",
+      "CUDA is  available\n",
+      "MPS not available because the current PyTorch install was not built with MPS enabled.\n",
+      "CUDA is available, we have found  1  GPU(s)\n",
+      "NVIDIA GeForce RTX 4090\n",
+      "CUDA version: 12.1\n",
+      "hf_embeddings_device_type: cuda:0\n",
+      "hf_pipeline_device_type: cuda:0\n",
+      "load INSTRUCTOR_Transformer\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n",
+      "max_seq_length  512\n",
+      "Completed in 0.427s\n",
+      "Load index from ./data/faiss_1024_512/ with FAISS\n",
+      "Completed in 0.014s\n",
+      "initializing LLM: openai\n",
+      "  hf_pipeline_device_type: cuda:0\n",
+      "     load_quantized_model: None\n",
+      "              torch_dtype: torch.bfloat16\n",
+      "                 n_threds: 4\n",
+      "              using model: gpt-3.5-turbo\n",
+      "initialization complete\n",
+      "Completed in 0.024s\n",
+      "\n",
+      "Question: What's PCI DSS?\n",
+      "{'question': \"What's PCI DSS?\", 'chat_history': []}\n",
+      "PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that outlines the technical and operational requirements for protecting payment card data. The standard provides guidelines and security controls to help organizations build and maintain a secure network, protect account data, maintain a vulnerability management program, and implement strong access control measures. Compliance with PCI DSS is required for any organization that handles payment card data.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 4.178s\n",
+      "\n",
+      "Question: Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?\n",
+      "{'question': 'Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that outlines the technical and operational requirements for protecting payment card data. The standard provides guidelines and security controls to help organizations build and maintain a secure network, protect account data, maintain a vulnerability management program, and implement strong access control measures. Compliance with PCI DSS is required for any organization that handles payment card data.')]}\n",
+      "What are the differences between PCI DSS version 3.2.1 and version 4.0?\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "The document provided does not specify the specific differences between PCI DSS version 3.2.1 and version 4.0. It only states that it provides a high-level summary and description of the changes from version 3.2.1 to version 4.0. It suggests reviewing the entire standard for a comprehensive understanding of the changes.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 2.636s\n",
+      "\n",
+      "Question: new requirements for vulnerability assessments\n",
+      "{'question': 'new requirements for vulnerability assessments', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that outlines the technical and operational requirements for protecting payment card data. The standard provides guidelines and security controls to help organizations build and maintain a secure network, protect account data, maintain a vulnerability management program, and implement strong access control measures. Compliance with PCI DSS is required for any organization that handles payment card data.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'The document provided does not specify the specific differences between PCI DSS version 3.2.1 and version 4.0. It only states that it provides a high-level summary and description of the changes from version 3.2.1 to version 4.0. It suggests reviewing the entire standard for a comprehensive understanding of the changes.')]}\n",
+      "What are the new requirements for vulnerability assessments in PCI DSS version 4.0?\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "In PCI DSS version 4.0, there are several new requirements for vulnerability assessments. Here are the new requirements:\n",
+      "\n",
+      "- Requirement 11.3.1.1: Manage all other applicable vulnerabilities (those not ranked as high-risk or critical).\n",
+      "- Requirement 11.3.1.2: Internal vulnerability scans are performed via authenticated scanning.\n",
+      "- Requirement A3.3.1: Failures of automated log review mechanisms and automated code review tools are detected, alerted, and reported in a timely manner.\n",
+      "\n",
+      "These new requirements are applicable to all entities and service providers and are effective immediately for all v4.0 assessments.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 3.809s\n",
+      "\n",
+      "Question: more on penetration testing\n",
+      "{'question': 'more on penetration testing', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that outlines the technical and operational requirements for protecting payment card data. The standard provides guidelines and security controls to help organizations build and maintain a secure network, protect account data, maintain a vulnerability management program, and implement strong access control measures. Compliance with PCI DSS is required for any organization that handles payment card data.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'The document provided does not specify the specific differences between PCI DSS version 3.2.1 and version 4.0. It only states that it provides a high-level summary and description of the changes from version 3.2.1 to version 4.0. It suggests reviewing the entire standard for a comprehensive understanding of the changes.'), ('new requirements for vulnerability assessments', 'In PCI DSS version 4.0, there are several new requirements for vulnerability assessments. Here are the new requirements:\\n\\n- Requirement 11.3.1.1: Manage all other applicable vulnerabilities (those not ranked as high-risk or critical).\\n- Requirement 11.3.1.2: Internal vulnerability scans are performed via authenticated scanning.\\n- Requirement A3.3.1: Failures of automated log review mechanisms and automated code review tools are detected, alerted, and reported in a timely manner.\\n\\nThese new requirements are applicable to all entities and service providers and are effective immediately for all v4.0 assessments.')]}\n",
+      "Can you provide more information about the changes related to penetration testing in PCI DSS version 4.0?\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "I'm sorry, but I don't have access to the specific changes related to penetration testing in PCI DSS version 4.0. The information provided does not include details about the changes made to penetration testing requirements. It is recommended to review the PCI DSS version 4.0 document in its entirety to understand the changes related to penetration testing.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 2.609s\n",
+      "Total time used: 13.232 s\n",
+      "Number of tokens generated: 425\n",
+      "Average generation speed: 32.119 tokens/s\n",
+      "CPU times: user 137 ms, sys: 76.1 ms, total: 213 ms\n",
+      "Wall time: 16.5 s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "os.environ[\"HF_RP\"] = \"1.05\"\n",
+    "os.environ[\"OPENAI_MODEL_NAME\"] = \"gpt-3.5-turbo\"\n",
+    "!python qa_chain_test.py"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "8a022a7b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading env vars from: /home/inflaton/code/chat-with-orca-2/.env\n",
+      "Running on:  Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.35\n",
+      "MPS is NOT available\n",
+      "CUDA is  available\n",
+      "MPS not available because the current PyTorch install was not built with MPS enabled.\n",
+      "CUDA is available, we have found  1  GPU(s)\n",
+      "NVIDIA GeForce RTX 4090\n",
+      "CUDA version: 12.1\n",
+      "hf_embeddings_device_type: cuda:0\n",
+      "hf_pipeline_device_type: cuda:0\n",
+      "load INSTRUCTOR_Transformer\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n",
+      "max_seq_length  512\n",
+      "Completed in 0.429s\n",
+      "Load index from ./data/faiss_1024_512/ with FAISS\n",
+      "Completed in 0.013s\n",
+      "initializing LLM: openai\n",
+      "  hf_pipeline_device_type: cuda:0\n",
+      "     load_quantized_model: None\n",
+      "              torch_dtype: torch.bfloat16\n",
+      "                 n_threds: 4\n",
+      "              using model: gpt-3.5-turbo-instruct\n",
+      "initialization complete\n",
+      "Completed in 0.024s\n",
+      "\n",
+      "Question: What's PCI DSS?\n",
+      "{'question': \"What's PCI DSS?\", 'chat_history': []}\n",
+      " The PCI Data Security Standard (PCI DSS) is a global standard that provides a baseline of technical and operational requirements designated to protect payment data. It is developed and managed by the PCI Security Standards Council (PCI SSC) and is constantly evolving to address new threats and vulnerabilities in the payment industry.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 4.007s\n",
+      "\n",
+      "Question: Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?\n",
+      "{'question': 'Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'chat_history': [(\"What's PCI DSS?\", 'The PCI Data Security Standard (PCI DSS) is a global standard that provides a baseline of technical and operational requirements designated to protect payment data. It is developed and managed by the PCI Security Standards Council (PCI SSC) and is constantly evolving to address new threats and vulnerabilities in the payment industry.')]}\n",
+      " What changes were made from PCI DSS version 3.2.1 to version 4.0?\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " The changes made from PCI DSS version 3.2.1 to version 4.0 include updates to align with the changes from PCI DSS v3.2.1 to v4.0, as well as minor corrections and edits for clarification and/or format. These changes can be found in the Summary of Changes document and include changes to the introductory sections, general changes to requirements, and additional changes per requirement.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 2.118s\n",
+      "\n",
+      "Question: new requirements for vulnerability assessments\n",
+      "{'question': 'new requirements for vulnerability assessments', 'chat_history': [(\"What's PCI DSS?\", 'The PCI Data Security Standard (PCI DSS) is a global standard that provides a baseline of technical and operational requirements designated to protect payment data. It is developed and managed by the PCI Security Standards Council (PCI SSC) and is constantly evolving to address new threats and vulnerabilities in the payment industry.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'The changes made from PCI DSS version 3.2.1 to version 4.0 include updates to align with the changes from PCI DSS v3.2.1 to v4.0, as well as minor corrections and edits for clarification and/or format. These changes can be found in the Summary of Changes document and include changes to the introductory sections, general changes to requirements, and additional changes per requirement.')]}\n",
+      " What are the new requirements for vulnerability assessments in PCI DSS version 4.0?\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " The new requirements for vulnerability assessments in PCI DSS version 4.0 include performing internal vulnerability scans via authenticated scanning, managing all other applicable vulnerabilities, and maintaining an inventory of bespoke and custom software. These requirements are effective immediately for all v4.0 assessments. Additionally, there is a new requirement for targeted risk analysis to determine the frequency of log reviews for all other system components, which will be effective until 31 March 2025.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 2.111s\n",
+      "\n",
+      "Question: more on penetration testing\n",
+      "{'question': 'more on penetration testing', 'chat_history': [(\"What's PCI DSS?\", 'The PCI Data Security Standard (PCI DSS) is a global standard that provides a baseline of technical and operational requirements designated to protect payment data. It is developed and managed by the PCI Security Standards Council (PCI SSC) and is constantly evolving to address new threats and vulnerabilities in the payment industry.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'The changes made from PCI DSS version 3.2.1 to version 4.0 include updates to align with the changes from PCI DSS v3.2.1 to v4.0, as well as minor corrections and edits for clarification and/or format. These changes can be found in the Summary of Changes document and include changes to the introductory sections, general changes to requirements, and additional changes per requirement.'), ('new requirements for vulnerability assessments', 'The new requirements for vulnerability assessments in PCI DSS version 4.0 include performing internal vulnerability scans via authenticated scanning, managing all other applicable vulnerabilities, and maintaining an inventory of bespoke and custom software. These requirements are effective immediately for all v4.0 assessments. Additionally, there is a new requirement for targeted risk analysis to determine the frequency of log reviews for all other system components, which will be effective until 31 March 2025.')]}\n",
+      " Can you provide more information about the new requirements for penetration testing in PCI DSS version 4.0?\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      " The new requirements for penetration testing in PCI DSS version 4.0 include defining and documenting a penetration testing methodology, using industry-accepted approaches, testing the entire CDE perimeter and critical systems, testing from both inside and outside the network, and conducting application-layer and network-layer penetration tests. Additionally, there are new requirements for maintaining an inventory of trusted keys and certificates, and confirming the validity of certificates used for PAN transmissions over open, public networks. These requirements are effective immediately for all v4.0 assessments and are considered evolving requirements until March 31, 2025.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 2.124s\n",
+      "Total time used: 10.360 s\n",
+      "Number of tokens generated: 411\n",
+      "Average generation speed: 39.673 tokens/s\n",
+      "CPU times: user 120 ms, sys: 58.6 ms, total: 178 ms\n",
+      "Wall time: 13.6 s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "os.environ[\"HF_RP\"] = \"1.10\"\n",
+    "os.environ[\"OPENAI_MODEL_NAME\"] = \"gpt-3.5-turbo-instruct\"\n",
+    "!python qa_chain_test.py"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "db48dcc4",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading env vars from: /home/inflaton/code/chat-with-orca-2/.env\n",
+      "Running on:  Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.35\n",
+      "MPS is NOT available\n",
+      "CUDA is  available\n",
+      "MPS not available because the current PyTorch install was not built with MPS enabled.\n",
+      "CUDA is available, we have found  1  GPU(s)\n",
+      "NVIDIA GeForce RTX 4090\n",
+      "CUDA version: 12.1\n",
+      "hf_embeddings_device_type: cuda:0\n",
+      "hf_pipeline_device_type: cuda:0\n",
+      "load INSTRUCTOR_Transformer\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n",
+      "max_seq_length  512\n",
+      "Completed in 0.391s\n",
+      "Load index from ./data/faiss_1024_512/ with FAISS\n",
+      "Completed in 0.013s\n",
+      "initializing LLM: openai\n",
+      "  hf_pipeline_device_type: cuda:0\n",
+      "     load_quantized_model: None\n",
+      "              torch_dtype: torch.bfloat16\n",
+      "                 n_threds: 4\n",
+      "              using model: gpt-4\n",
+      "initialization complete\n",
+      "Completed in 0.026s\n",
+      "\n",
+      "Question: What's PCI DSS?\n",
+      "{'question': \"What's PCI DSS?\", 'chat_history': []}\n",
+      "The PCI Data Security Standard (PCI DSS) is a global standard that provides a baseline of technical and operational requirements designed to protect payment account data. It includes requirements for building and maintaining a secure network, protecting account data, maintaining a vulnerability management program, and implementing strong access control measures. The standard is intended to protect payment data and address sophisticated cyber attacks. It is managed by the PCI Security Standards Council.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 7.688s\n",
+      "\n",
+      "Question: Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?\n",
+      "{'question': 'Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'chat_history': [(\"What's PCI DSS?\", 'The PCI Data Security Standard (PCI DSS) is a global standard that provides a baseline of technical and operational requirements designed to protect payment account data. It includes requirements for building and maintaining a secure network, protecting account data, maintaining a vulnerability management program, and implementing strong access control measures. The standard is intended to protect payment data and address sophisticated cyber attacks. It is managed by the PCI Security Standards Council.')]}\n",
+      "Can you provide a summary of the changes that were made from version 3.2.1 to version 4.0 of the PCI Data Security Standard (PCI DSS)?\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "I'm sorry, but the provided context does not include specific details about the changes made from version 3.2.1 to version 4.0 of the PCI Data Security Standard (PCI DSS). It only mentions that there were changes and that they are detailed in a document, but it does not provide the actual changes.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 6.529s\n",
+      "\n",
+      "Question: new requirements for vulnerability assessments\n",
+      "{'question': 'new requirements for vulnerability assessments', 'chat_history': [(\"What's PCI DSS?\", 'The PCI Data Security Standard (PCI DSS) is a global standard that provides a baseline of technical and operational requirements designed to protect payment account data. It includes requirements for building and maintaining a secure network, protecting account data, maintaining a vulnerability management program, and implementing strong access control measures. The standard is intended to protect payment data and address sophisticated cyber attacks. It is managed by the PCI Security Standards Council.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', \"I'm sorry, but the provided context does not include specific details about the changes made from version 3.2.1 to version 4.0 of the PCI Data Security Standard (PCI DSS). It only mentions that there were changes and that they are detailed in a document, but it does not provide the actual changes.\")]}\n",
+      "What are the new requirements for vulnerability assessments in the PCI DSS?\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "The new requirements for vulnerability assessments in the PCI DSS v4.0 as compared to v3.2.1 are:\n",
+      "\n",
+      "1. Requirement 10.4.2.1: A targeted risk analysis is performed to determine the frequency of log reviews for all other system components.\n",
+      "\n",
+      "2. Requirement 11.3.1.1: Manage all other applicable vulnerabilities (those not ranked as high-risk or critical).\n",
+      "\n",
+      "3. Requirement 11.3.1.2: Internal vulnerability scans are performed via authenticated scanning.\n",
+      "\n",
+      "4. Requirement A1.2.3: The multi-tenant service provider implements processes or mechanisms for reporting and addressing suspected or confirmed security incidents and vulnerabilities.\n",
+      "\n",
+      "5. Requirement A3.3.1: Failures of the following are detected, alerted, and reported in a timely manner: Automated log review mechanisms and Automated code review tools.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 11.546s\n",
+      "\n",
+      "Question: more on penetration testing\n",
+      "{'question': 'more on penetration testing', 'chat_history': [(\"What's PCI DSS?\", 'The PCI Data Security Standard (PCI DSS) is a global standard that provides a baseline of technical and operational requirements designed to protect payment account data. It includes requirements for building and maintaining a secure network, protecting account data, maintaining a vulnerability management program, and implementing strong access control measures. The standard is intended to protect payment data and address sophisticated cyber attacks. It is managed by the PCI Security Standards Council.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', \"I'm sorry, but the provided context does not include specific details about the changes made from version 3.2.1 to version 4.0 of the PCI Data Security Standard (PCI DSS). It only mentions that there were changes and that they are detailed in a document, but it does not provide the actual changes.\"), ('new requirements for vulnerability assessments', 'The new requirements for vulnerability assessments in the PCI DSS v4.0 as compared to v3.2.1 are:\\n\\n1. Requirement 10.4.2.1: A targeted risk analysis is performed to determine the frequency of log reviews for all other system components.\\n\\n2. Requirement 11.3.1.1: Manage all other applicable vulnerabilities (those not ranked as high-risk or critical).\\n\\n3. Requirement 11.3.1.2: Internal vulnerability scans are performed via authenticated scanning.\\n\\n4. Requirement A1.2.3: The multi-tenant service provider implements processes or mechanisms for reporting and addressing suspected or confirmed security incidents and vulnerabilities.\\n\\n5. Requirement A3.3.1: Failures of the following are detected, alerted, and reported in a timely manner: Automated log review mechanisms and Automated code review tools.')]}\n",
+      "Can you provide more information on penetration testing in the context of PCI DSS?\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Penetration testing in the context of PCI DSS (Payment Card Industry Data Security Standard) is a requirement for entities that handle cardholder data. According to the PCI DSS v4.0 ROC Template, Requirement 11.4 states that external and internal penetration testing must be regularly performed, and any exploitable vulnerabilities and security weaknesses identified must be corrected.\n",
+      "\n",
+      "The penetration testing methodology should be defined, documented, and implemented by the entity. It should include industry-accepted penetration testing approaches and cover the entire Cardholder Data Environment (CDE) perimeter and critical systems. The testing should be conducted from both inside and outside the network.\n",
+      "\n",
+      "The testing should also validate any segmentation and scope-reduction controls. It should include application-layer penetration testing to identify, at a minimum, the vulnerabilities listed in Requirement 6.2.4. Network-layer penetration tests should encompass all components that support network functions as well as operating systems.\n",
+      "\n",
+      "If exploitable vulnerabilities and security weaknesses are found during penetration testing, they should be corrected in accordance with the entity's assessment of the risk posed by the security issue as defined in Requirement 6.3.1. The penetration testing should then be repeated to verify the corrections.\n",
+      "\n",
+      "The external penetration test should be performed by a qualified internal resource or a qualified external third-party, ensuring that organizational independence of the tester exists.\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 16.494s\n",
+      "Total time used: 42.257 s\n",
+      "Number of tokens generated: 670\n",
+      "Average generation speed: 15.855 tokens/s\n",
+      "CPU times: user 356 ms, sys: 177 ms, total: 533 ms\n",
+      "Wall time: 45.5 s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "os.environ[\"HF_RP\"] = \"1.15\"\n",
+    "os.environ[\"OPENAI_MODEL_NAME\"] = \"gpt-4\"\n",
+    "!python qa_chain_test.py"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebook/nvidia-4090-orca-2-13b.ipynb

@@ -0,0 +1,415 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "874cd58e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "workding dir: /home/inflaton/code/chat-with-orca-2\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import sys\n",
+    "from pathlib import Path\n",
+    "\n",
+    "workding_dir = str(Path.cwd().parent)\n",
+    "os.chdir(workding_dir)\n",
+    "sys.path.append(workding_dir)\n",
+    "print(\"workding dir:\", workding_dir)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "19f967ca",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "os.environ[\"LLM_MODEL_TYPE\"] = \"huggingface\"\n",
+    "os.environ[\"HUGGINGFACE_MODEL_NAME_OR_PATH\"] = \"microsoft/Orca-2-13b\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "a6d96660",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading env vars from: /home/inflaton/code/chat-with-orca-2/.env\n",
+      "Running on:  Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.35\n",
+      "MPS is NOT available\n",
+      "CUDA is  available\n",
+      "MPS not available because the current PyTorch install was not built with MPS enabled.\n",
+      "CUDA is available, we have found  1  GPU(s)\n",
+      "NVIDIA GeForce RTX 4090\n",
+      "CUDA version: 12.1\n",
+      "hf_embeddings_device_type: cuda:0\n",
+      "hf_pipeline_device_type: cuda:0\n",
+      "load INSTRUCTOR_Transformer\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n",
+      "max_seq_length  512\n",
+      "Completed in 0.427s\n",
+      "Load index from ./data/faiss_1024_512/ with FAISS\n",
+      "Completed in 0.013s\n",
+      "initializing LLM: huggingface\n",
+      "  hf_pipeline_device_type: cuda:0\n",
+      "     load_quantized_model: None\n",
+      "              torch_dtype: torch.bfloat16\n",
+      "                 n_threds: 4\n",
+      "            loading model: microsoft/Orca-2-13b\n",
+      "            HF auth token: None\n",
+      "       repetition_penalty: 1.05\n",
+      "Loading checkpoint shards: 100%|██████████████████| 6/6 [00:16<00:00,  2.72s/it]\n",
+      "initialization complete\n",
+      "Completed in 33.618s\n",
+      "\n",
+      "Question: What's PCI DSS?\n",
+      "{'question': \"What's PCI DSS?\", 'chat_history': []}\n",
+      "PCI DSS is a global standard that provides a baseline of technical and operational requirements designed to protect payment account data. It helps ensure the security of cardholder data and reduces the risk of cyberattacks.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 49.719s\n",
+      "\n",
+      "Question: Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?\n",
+      "{'question': 'Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS is a global standard that provides a baseline of technical and operational requirements designed to protect payment account data. It helps ensure the security of cardholder data and reduces the risk of cyberattacks.')]}\n",
+      "¿Puedes resumir los cambios realizados desde la versión 3.2.1 hasta la versión 4.0 de PCI DSS?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Sí, puedo resumir los cambios realizados desde la versión 3.2.1 hasta la versión 4.0 de PCI DSS. Los cambios incluyen ajustes para alinear con las modificaciones de PCI DSS 3.2 a PCI DSS 3.2.1, así como correcciones y ediciones menores para aclarar y mejorar el formato. Luego, en la versión 4.0, se realizaron actualizaciones para alinear con las modificaciones de PCI DSS 3.2.1 a PCI DSS 4.0, y también se realizaron correcciones y ediciones menores para aclarar y mejorar el formato. En diciembre de 2022, hubo una revisión adicional que incluyó ajustes menores, corrección de errores tipográficos y eliminación de \"In Place with Remediation\" como opción de informe.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 144.641s\n",
+      "\n",
+      "Question: new requirements for vulnerability assessments\n",
+      "{'question': 'new requirements for vulnerability assessments', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS is a global standard that provides a baseline of technical and operational requirements designed to protect payment account data. It helps ensure the security of cardholder data and reduces the risk of cyberattacks.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'Sí, puedo resumir los cambios realizados desde la versión 3.2.1 hasta la versión 4.0 de PCI DSS. Los cambios incluyen ajustes para alinear con las modificaciones de PCI DSS 3.2 a PCI DSS 3.2.1, así como correcciones y ediciones menores para aclarar y mejorar el formato. Luego, en la versión 4.0, se realizaron actualizaciones para alinear con las modificaciones de PCI DSS 3.2.1 a PCI DSS 4.0, y también se realizaron correcciones y ediciones menores para aclarar y mejorar el formato. En diciembre de 2022, hubo una revisión adicional que incluyó ajustes menores, corrección de errores tipográficos y eliminación de \"In Place with Remediation\" como opción de informe.')]}\n",
+      "¿Cuáles son las nuevas requisitos para las evaluaciones de vulnerabilidades en la versión 4.0 de PCI DSS?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Las nuevas requisitos para las evaluaciones de vulnerabilidades en la versión 4.0 de PCI DSS incluyen resolver vulnerabilidades con una puntuación de 4.0 o más en el CVSS, realizar rescans cuando sea necesario, y tener un método de penetration testing definido, documentado e implementado que incluye enfoques aceptados por la industria y cobertura para el perímetro del SDE y los sistemas críticos.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 87.622s\n",
+      "\n",
+      "Question: more on penetration testing\n",
+      "{'question': 'more on penetration testing', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS is a global standard that provides a baseline of technical and operational requirements designed to protect payment account data. It helps ensure the security of cardholder data and reduces the risk of cyberattacks.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'Sí, puedo resumir los cambios realizados desde la versión 3.2.1 hasta la versión 4.0 de PCI DSS. Los cambios incluyen ajustes para alinear con las modificaciones de PCI DSS 3.2 a PCI DSS 3.2.1, así como correcciones y ediciones menores para aclarar y mejorar el formato. Luego, en la versión 4.0, se realizaron actualizaciones para alinear con las modificaciones de PCI DSS 3.2.1 a PCI DSS 4.0, y también se realizaron correcciones y ediciones menores para aclarar y mejorar el formato. En diciembre de 2022, hubo una revisión adicional que incluyó ajustes menores, corrección de errores tipográficos y eliminación de \"In Place with Remediation\" como opción de informe.'), ('new requirements for vulnerability assessments', 'Las nuevas requisitos para las evaluaciones de vulnerabilidades en la versión 4.0 de PCI DSS incluyen resolver vulnerabilidades con una puntuación de 4.0 o más en el CVSS, realizar rescans cuando sea necesario, y tener un método de penetration testing definido, documentado e implementado que incluye enfoques aceptados por la industria y cobertura para el perímetro del SDE y los sistemas críticos.')]}\n",
+      "¿Puedes dar más detalles sobre las prácticas de prueba de penetración en PCI DSS?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Sí, puedo dar más detalles sobre las prácticas de prueba de penetración en PCI DSS. La PCI DSS requiere que se realice un penetration testing externo al menos una vez cada 12 meses y después de cualquier modificación o cambio significativo en la infraestructura o el software. El penetration testing debe ser realizado por un recurso interno calificado o un tercero de fuera de la organización calificado y debe cumplir con la metodología definida por la entidad. Además, las vulnerabilidades y debilidades de seguridad que se encuentren durante el penetration testing deben ser corregidas y verificadas mediante la repetición del testing.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 115.566s\n",
+      "Total time used: 397.548 s\n",
+      "Number of tokens generated: 641\n",
+      "Average generation speed: 1.612 tokens/s\n",
+      "CPU times: user 3.51 s, sys: 1.02 s, total: 4.53 s\n",
+      "Wall time: 7min 16s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "os.environ[\"HF_RP\"] = \"1.05\"\n",
+    "!python qa_chain_test.py"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "8a022a7b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading env vars from: /home/inflaton/code/chat-with-orca-2/.env\n",
+      "Running on:  Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.35\n",
+      "MPS is NOT available\n",
+      "CUDA is  available\n",
+      "MPS not available because the current PyTorch install was not built with MPS enabled.\n",
+      "CUDA is available, we have found  1  GPU(s)\n",
+      "NVIDIA GeForce RTX 4090\n",
+      "CUDA version: 12.1\n",
+      "hf_embeddings_device_type: cuda:0\n",
+      "hf_pipeline_device_type: cuda:0\n",
+      "load INSTRUCTOR_Transformer\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n",
+      "max_seq_length  512\n",
+      "Completed in 1.001s\n",
+      "Load index from ./data/faiss_1024_512/ with FAISS\n",
+      "Completed in 0.021s\n",
+      "initializing LLM: huggingface\n",
+      "  hf_pipeline_device_type: cuda:0\n",
+      "     load_quantized_model: None\n",
+      "              torch_dtype: torch.bfloat16\n",
+      "                 n_threds: 4\n",
+      "            loading model: microsoft/Orca-2-13b\n",
+      "            HF auth token: None\n",
+      "       repetition_penalty: 1.1\n",
+      "Loading checkpoint shards: 100%|██████████████████| 6/6 [00:16<00:00,  2.75s/it]\n",
+      "initialization complete\n",
+      "Completed in 63.590s\n",
+      "\n",
+      "Question: What's PCI DSS?\n",
+      "{'question': \"What's PCI DSS?\", 'chat_history': []}\n",
+      "PCI DSS is a set of requirements designed to protect payment account data. It provides a baseline of technical and operational requirements for organizations that handle payment card transactions.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 33.852s\n",
+      "\n",
+      "Question: Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?\n",
+      "{'question': 'Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS is a set of requirements designed to protect payment account data. It provides a baseline of technical and operational requirements for organizations that handle payment card transactions.')]}\n",
+      "¿Puedes resumir los cambios realizados desde la versión 3.2.1 hasta la versión 4.0 de PCI DSS?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Sí, puedo resumir los cambios realizados desde la versión 3.2.1 hasta la versión 4.0 de PCI DSS. Los cambios incluyen ajustes para alinear con las modificaciones entre las versiones 3.2 y 3.2.1, así como correcciones menores y ediciones para aclarar y mejorar el formato. Además, hay actualizaciones para adaptarse a las nuevas exigencias y mejoras en la seguridad.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 78.279s\n",
+      "\n",
+      "Question: new requirements for vulnerability assessments\n",
+      "{'question': 'new requirements for vulnerability assessments', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS is a set of requirements designed to protect payment account data. It provides a baseline of technical and operational requirements for organizations that handle payment card transactions.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'Sí, puedo resumir los cambios realizados desde la versión 3.2.1 hasta la versión 4.0 de PCI DSS. Los cambios incluyen ajustes para alinear con las modificaciones entre las versiones 3.2 y 3.2.1, así como correcciones menores y ediciones para aclarar y mejorar el formato. Además, hay actualizaciones para adaptarse a las nuevas exigencias y mejoras en la seguridad.')]}\n",
+      "¿Cuáles son las nuevas exigencias para las evaluaciones de vulnerabilidades en la versión 4.0 de PCI DSS?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Las nuevas exigencias para las evaluaciones de vulnerabilidades en la versión 4.0 de PCI DSS incluyen realizar escaneos de vulnerabilidad externa después de cualquier cambio significativo y resolver las vulnerabilidades que sean de nivel 4.0 o superior según el CVSS. También es necesario realizar pruebas de penetración externa e interna regularmente y corregir las vulnerabilidades y debilidades de seguridad que se encuentren. Además, se deben instalar parches de seguridad y actualizaciones de sistemas en un plazo no mayor a un mes para las críticas o altas y en un plazo no mayor a tres meses para el resto.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 102.950s\n",
+      "\n",
+      "Question: more on penetration testing\n",
+      "{'question': 'more on penetration testing', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS is a set of requirements designed to protect payment account data. It provides a baseline of technical and operational requirements for organizations that handle payment card transactions.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'Sí, puedo resumir los cambios realizados desde la versión 3.2.1 hasta la versión 4.0 de PCI DSS. Los cambios incluyen ajustes para alinear con las modificaciones entre las versiones 3.2 y 3.2.1, así como correcciones menores y ediciones para aclarar y mejorar el formato. Además, hay actualizaciones para adaptarse a las nuevas exigencias y mejoras en la seguridad.'), ('new requirements for vulnerability assessments', 'Las nuevas exigencias para las evaluaciones de vulnerabilidades en la versión 4.0 de PCI DSS incluyen realizar escaneos de vulnerabilidad externa después de cualquier cambio significativo y resolver las vulnerabilidades que sean de nivel 4.0 o superior según el CVSS. También es necesario realizar pruebas de penetración externa e interna regularmente y corregir las vulnerabilidades y debilidades de seguridad que se encuentren. Además, se deben instalar parches de seguridad y actualizaciones de sistemas en un plazo no mayor a un mes para las críticas o altas y en un plazo no mayor a tres meses para el resto.')]}\n",
+      "¿Puedo obtener más información sobre las pruebas de penetración en PCI DSS?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Yes, you can get more information about penetration tests in PCI DSS from the provided context. Look for the sections related to 11.4.3 External Penetration Testing and 11.4.4 Exploitable Vulnerabilities and Security Weaknesses Found During Penetration Testing.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 57.811s\n",
+      "Total time used: 272.891 s\n",
+      "Number of tokens generated: 478\n",
+      "Average generation speed: 1.752 tokens/s\n",
+      "CPU times: user 2.79 s, sys: 824 ms, total: 3.61 s\n",
+      "Wall time: 5min 42s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "os.environ[\"HF_RP\"] = \"1.10\"\n",
+    "!python qa_chain_test.py"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "db48dcc4",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading env vars from: /home/inflaton/code/chat-with-orca-2/.env\n",
+      "Running on:  Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.35\n",
+      "MPS is NOT available\n",
+      "CUDA is  available\n",
+      "MPS not available because the current PyTorch install was not built with MPS enabled.\n",
+      "CUDA is available, we have found  1  GPU(s)\n",
+      "NVIDIA GeForce RTX 4090\n",
+      "CUDA version: 12.1\n",
+      "hf_embeddings_device_type: cuda:0\n",
+      "hf_pipeline_device_type: cuda:0\n",
+      "load INSTRUCTOR_Transformer\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n",
+      "max_seq_length  512\n",
+      "Completed in 1.066s\n",
+      "Load index from ./data/faiss_1024_512/ with FAISS\n",
+      "Completed in 0.024s\n",
+      "initializing LLM: huggingface\n",
+      "  hf_pipeline_device_type: cuda:0\n",
+      "     load_quantized_model: None\n",
+      "              torch_dtype: torch.bfloat16\n",
+      "                 n_threds: 4\n",
+      "            loading model: microsoft/Orca-2-13b\n",
+      "            HF auth token: None\n",
+      "       repetition_penalty: 1.15\n",
+      "Loading checkpoint shards: 100%|██████████████████| 6/6 [00:19<00:00,  3.26s/it]\n",
+      "initialization complete\n",
+      "Completed in 28.543s\n",
+      "\n",
+      "Question: What's PCI DSS?\n",
+      "{'question': \"What's PCI DSS?\", 'chat_history': []}\n",
+      "PCI DSS is a set of requirements designed to protect payment account data. It includes guidelines for building and maintaining secure networks and systems, protecting account data, managing vulnerabilities, implementing strong access control measures, and more.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 40.285s\n",
+      "\n",
+      "Question: Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?\n",
+      "{'question': 'Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS is a set of requirements designed to protect payment account data. It includes guidelines for building and maintaining secure networks and systems, protecting account data, managing vulnerabilities, implementing strong access control measures, and more.')]}\n",
+      "¿Puedes resumir los cambios realizados desde la versión 3.2.1 hasta la versión 4.0 de PCI DSS?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Sí, puedo resumir los cambios realizados desde la versión 3.2.1 hasta la versión 4.0 de PCI DSS. Los cambios incluyen ajustes para alinear con las modificaciones entre las versiones 3.2 y 3.2.1, así como actualizaciones para alinearse con las versiones 3.2.1 y 4.0. También hay correcciones menores y ediciones para mejorar la claridad y el formato del documento. Algunos ejemplos específicos de cambios incluyen la eliminación de \"In Place with Remediation\" como una opción de informe y ajustes en las requisitos 3.6.3 y 6.3.3.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 106.276s\n",
+      "\n",
+      "Question: new requirements for vulnerability assessments\n",
+      "{'question': 'new requirements for vulnerability assessments', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS is a set of requirements designed to protect payment account data. It includes guidelines for building and maintaining secure networks and systems, protecting account data, managing vulnerabilities, implementing strong access control measures, and more.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'Sí, puedo resumir los cambios realizados desde la versión 3.2.1 hasta la versión 4.0 de PCI DSS. Los cambios incluyen ajustes para alinear con las modificaciones entre las versiones 3.2 y 3.2.1, así como actualizaciones para alinearse con las versiones 3.2.1 y 4.0. También hay correcciones menores y ediciones para mejorar la claridad y el formato del documento. Algunos ejemplos específicos de cambios incluyen la eliminación de \"In Place with Remediation\" como una opción de informe y ajustes en las requisitos 3.6.3 y 6.3.3.')]}\n",
+      "¿Cuáles son las nuevas recomendaciones para las evaluaciones de vulnerabilidades en la versión 4.0 de PCI DSS?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Las nuevas recomendaciones para las evaluaciones de vulnerabilidades en la versión 4.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 38.122s\n",
+      "\n",
+      "Question: more on penetration testing\n",
+      "{'question': 'more on penetration testing', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS is a set of requirements designed to protect payment account data. It includes guidelines for building and maintaining secure networks and systems, protecting account data, managing vulnerabilities, implementing strong access control measures, and more.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'Sí, puedo resumir los cambios realizados desde la versión 3.2.1 hasta la versión 4.0 de PCI DSS. Los cambios incluyen ajustes para alinear con las modificaciones entre las versiones 3.2 y 3.2.1, así como actualizaciones para alinearse con las versiones 3.2.1 y 4.0. También hay correcciones menores y ediciones para mejorar la claridad y el formato del documento. Algunos ejemplos específicos de cambios incluyen la eliminación de \"In Place with Remediation\" como una opción de informe y ajustes en las requisitos 3.6.3 y 6.3.3.'), ('new requirements for vulnerability assessments', 'Las nuevas recomendaciones para las evaluaciones de vulnerabilidades en la versión 4.')]}\n",
+      "¿Puedes dar más información sobre las pruebas de penetración en PCI DSS?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Sí, puedo dar más información sobre las pruebas de penetración en PCI DSS. La PCI DSS requiere que se realice un penetration testing externo al menos una vez cada doce meses y después de cualquier modificación o cambio significativo en la infraestructura o el software. El penetration testing debe ser realizado por un recurso interno calificado o un tercero de fuera de la empresa que tenga independencia orgánica. Las vulnerabilidades y debilidades de seguridad que se encuentren durante el penetration testing deben ser corregidas según los procedimientos establecidos en la Requisito 6.3.1 y se debe repetir el penetration testing para verificar las correcciones.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 106.927s\n",
+      "Total time used: 291.610 s\n",
+      "Number of tokens generated: 514\n",
+      "Average generation speed: 1.763 tokens/s\n",
+      "CPU times: user 2.33 s, sys: 1.12 s, total: 3.45 s\n",
+      "Wall time: 5min 26s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "os.environ[\"HF_RP\"] = \"1.15\"\n",
+    "!python qa_chain_test.py"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebook/nvidia-4090-orca-2-7b.ipynb

@@ -0,0 +1,439 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "874cd58e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "workding dir: /home/inflaton/code/chat-with-orca-2\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import sys\n",
+    "from pathlib import Path\n",
+    "\n",
+    "workding_dir = str(Path.cwd().parent)\n",
+    "os.chdir(workding_dir)\n",
+    "sys.path.append(workding_dir)\n",
+    "print(\"workding dir:\", workding_dir)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "19f967ca",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "os.environ[\"LLM_MODEL_TYPE\"] = \"huggingface\"\n",
+    "os.environ[\"HUGGINGFACE_MODEL_NAME_OR_PATH\"] = \"microsoft/Orca-2-7b\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "a6d96660",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading env vars from: /home/inflaton/code/chat-with-orca-2/.env\n",
+      "Running on:  Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.35\n",
+      "MPS is NOT available\n",
+      "CUDA is  available\n",
+      "MPS not available because the current PyTorch install was not built with MPS enabled.\n",
+      "CUDA is available, we have found  1  GPU(s)\n",
+      "NVIDIA GeForce RTX 4090\n",
+      "CUDA version: 12.1\n",
+      "hf_embeddings_device_type: cuda:0\n",
+      "hf_pipeline_device_type: cuda:0\n",
+      "load INSTRUCTOR_Transformer\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n",
+      "max_seq_length  512\n",
+      "Completed in 0.428s\n",
+      "Load index from ./data/faiss_1024_512/ with FAISS\n",
+      "Completed in 0.013s\n",
+      "initializing LLM: huggingface\n",
+      "  hf_pipeline_device_type: cuda:0\n",
+      "     load_quantized_model: None\n",
+      "              torch_dtype: torch.bfloat16\n",
+      "                 n_threds: 4\n",
+      "            loading model: microsoft/Orca-2-7b\n",
+      "            HF auth token: None\n",
+      "       repetition_penalty: 1.05\n",
+      "Loading checkpoint shards: 100%|██████████████████| 3/3 [00:08<00:00,  2.71s/it]\n",
+      "initialization complete\n",
+      "Completed in 12.199s\n",
+      "\n",
+      "Question: What's PCI DSS?\n",
+      "{'question': \"What's PCI DSS?\", 'chat_history': []}\n",
+      "PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment data.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 33.511s\n",
+      "\n",
+      "Question: Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?\n",
+      "{'question': 'Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment data.')]}\n",
+      "What are the main differences between PCI DSS version 3.2.1 and version 4.0?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "The main differences between PCI DSS version 3.2.1 and version 4.0 include changes to the requirements, testing procedures, and guidance. The standard has been updated to address new risks and vulnerabilities, as well as to reflect advances in technology and payment card industry practices. Some specific changes include updates to Requirement 6.3.3, the addition of a description for the change made to Requirement 8.3.9, and the addition of a description of the change made to Requirement 3.6.1.1. Additionally, the standard has been revised to align with the changes from PCI DSS v3.2 to PCI DSS v3.2.1 and to incorporate minor corrections and edits made for clarification and/or format purposes.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 4.836s\n",
+      "\n",
+      "Question: new requirements for vulnerability assessments\n",
+      "{'question': 'new requirements for vulnerability assessments', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment data.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'The main differences between PCI DSS version 3.2.1 and version 4.0 include changes to the requirements, testing procedures, and guidance. The standard has been updated to address new risks and vulnerabilities, as well as to reflect advances in technology and payment card industry practices. Some specific changes include updates to Requirement 6.3.3, the addition of a description for the change made to Requirement 8.3.9, and the addition of a description of the change made to Requirement 3.6.1.1. Additionally, the standard has been revised to align with the changes from PCI DSS v3.2 to PCI DSS v3.2.1 and to incorporate minor corrections and edits made for clarification and/or format purposes.')]}\n",
+      "What are the new requirements for vulnerability assessments in PCI DSS version 4.0?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "The new requirements for vulnerability assessments in PCI DSS version 4.0 include:\n",
+      "\n",
+      "- Requirement 4.1.2, which states that there is a new requirement for roles and responsibilities related to vulnerability assessments.\n",
+      "- Requirement 4.2.1, which requires confirmation that certificates used for Point-to-Point (PAN) transmissions over open, public networks are valid and not expired or revoked. This bullet is a best practice until March 31, 2025.\n",
+      "- Requirement 4.2.1.1, which introduces a new requirement to maintain an inventory of trusted keys and certificates. This is also a best practice until March 31, 2025.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 4.869s\n",
+      "\n",
+      "Question: more on penetration testing\n",
+      "{'question': 'more on penetration testing', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment data.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'The main differences between PCI DSS version 3.2.1 and version 4.0 include changes to the requirements, testing procedures, and guidance. The standard has been updated to address new risks and vulnerabilities, as well as to reflect advances in technology and payment card industry practices. Some specific changes include updates to Requirement 6.3.3, the addition of a description for the change made to Requirement 8.3.9, and the addition of a description of the change made to Requirement 3.6.1.1. Additionally, the standard has been revised to align with the changes from PCI DSS v3.2 to PCI DSS v3.2.1 and to incorporate minor corrections and edits made for clarification and/or format purposes.'), ('new requirements for vulnerability assessments', 'The new requirements for vulnerability assessments in PCI DSS version 4.0 include:\\n\\n- Requirement 4.1.2, which states that there is a new requirement for roles and responsibilities related to vulnerability assessments.\\n- Requirement 4.2.1, which requires confirmation that certificates used for Point-to-Point (PAN) transmissions over open, public networks are valid and not expired or revoked. This bullet is a best practice until March 31, 2025.\\n- Requirement 4.2.1.1, which introduces a new requirement to maintain an inventory of trusted keys and certificates. This is also a best practice until March 31, 2025.')]}\n",
+      "What are the new requirements for penetration testing in PCI DSS version 4.0?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "The new requirements for penetration testing in PCI DSS version 4.0 include defining a penetration testing methodology, covering the entire CDE perimeter and critical systems, testing from both inside and outside the network, testing to validate any segmentation and scope-reduction controls, and performing application-layer and network-layer penetration testing.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 2.903s\n",
+      "Total time used: 46.121 s\n",
+      "Number of tokens generated: 536\n",
+      "Average generation speed: 11.622 tokens/s\n",
+      "CPU times: user 561 ms, sys: 240 ms, total: 801 ms\n",
+      "Wall time: 1min 1s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "os.environ[\"HF_RP\"] = \"1.05\"\n",
+    "!python qa_chain_test.py"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "8a022a7b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading env vars from: /home/inflaton/code/chat-with-orca-2/.env\n",
+      "Running on:  Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.35\n",
+      "MPS is NOT available\n",
+      "CUDA is  available\n",
+      "MPS not available because the current PyTorch install was not built with MPS enabled.\n",
+      "CUDA is available, we have found  1  GPU(s)\n",
+      "NVIDIA GeForce RTX 4090\n",
+      "CUDA version: 12.1\n",
+      "hf_embeddings_device_type: cuda:0\n",
+      "hf_pipeline_device_type: cuda:0\n",
+      "load INSTRUCTOR_Transformer\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n",
+      "max_seq_length  512\n",
+      "Completed in 0.503s\n",
+      "Load index from ./data/faiss_1024_512/ with FAISS\n",
+      "Completed in 0.016s\n",
+      "initializing LLM: huggingface\n",
+      "  hf_pipeline_device_type: cuda:0\n",
+      "     load_quantized_model: None\n",
+      "              torch_dtype: torch.bfloat16\n",
+      "                 n_threds: 4\n",
+      "            loading model: microsoft/Orca-2-7b\n",
+      "            HF auth token: None\n",
+      "       repetition_penalty: 1.1\n",
+      "Loading checkpoint shards: 100%|██████████████████| 3/3 [00:13<00:00,  4.58s/it]\n",
+      "initialization complete\n",
+      "Completed in 17.662s\n",
+      "\n",
+      "Question: What's PCI DSS?\n",
+      "{'question': \"What's PCI DSS?\", 'chat_history': []}\n",
+      "PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment data.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 3.039s\n",
+      "\n",
+      "Question: Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?\n",
+      "{'question': 'Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment data.')]}\n",
+      "What are the main differences between PCI DSS versions 3.2.1 and 4.0?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "The main differences between PCI DSS versions 3.2.1 and 4.0 include changes to the introductory sections, general changes to the requirements, testing procedures, and guidance, and additional changes per requirement.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 1.769s\n",
+      "\n",
+      "Question: new requirements for vulnerability assessments\n",
+      "{'question': 'new requirements for vulnerability assessments', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment data.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'The main differences between PCI DSS versions 3.2.1 and 4.0 include changes to the introductory sections, general changes to the requirements, testing procedures, and guidance, and additional changes per requirement.')]}\n",
+      "What are the new requirements for vulnerability assessments in PCI DSS version 4.0?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "There are several new requirements related to vulnerability assessments in PCI DSS version 4.0. Some of them include:\n",
+      "\n",
+      "1. Requirement 4.1.2, which states that there should be clear roles and responsibilities defined for vulnerability management.\n",
+      "2. Requirement 4.2.1, which requires organizations to ensure that certificates used for point-to-point encryption (PAN) transmissions over open, public networks are valid and not expired or revoked. This is considered a best practice until March 31, 2025.\n",
+      "3. Requirement 10.4.2.1, which mandates a targeted risk analysis to determine the frequency of log reviews for all other system components.\n",
+      "4. Requirement 10.7.2 and 10.7.3, which require organizations to detect, alert, and address failures of critical security control systems promptly.\n",
+      "5. Requirement 11.3.1.1 and 11.3.1.2, which require internal vulnerability scans to be performed using authenticated scanning and to manage all applicable vulnerabilities, including those not ranked as high-risk or critical.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 7.887s\n",
+      "\n",
+      "Question: more on penetration testing\n",
+      "{'question': 'more on penetration testing', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a global standard that provides a baseline of technical and operational requirements designed to protect payment data.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'The main differences between PCI DSS versions 3.2.1 and 4.0 include changes to the introductory sections, general changes to the requirements, testing procedures, and guidance, and additional changes per requirement.'), ('new requirements for vulnerability assessments', 'There are several new requirements related to vulnerability assessments in PCI DSS version 4.0. Some of them include:\\n\\n1. Requirement 4.1.2, which states that there should be clear roles and responsibilities defined for vulnerability management.\\n2. Requirement 4.2.1, which requires organizations to ensure that certificates used for point-to-point encryption (PAN) transmissions over open, public networks are valid and not expired or revoked. This is considered a best practice until March 31, 2025.\\n3. Requirement 10.4.2.1, which mandates a targeted risk analysis to determine the frequency of log reviews for all other system components.\\n4. Requirement 10.7.2 and 10.7.3, which require organizations to detect, alert, and address failures of critical security control systems promptly.\\n5. Requirement 11.3.1.1 and 11.3.1.2, which require internal vulnerability scans to be performed using authenticated scanning and to manage all applicable vulnerabilities, including those not ranked as high-risk or critical.')]}\n",
+      "What are some key differences between penetration testing and vulnerability assessments in the context of PCI DSS?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Some key differences between penetration testing and vulnerability assessments in the context of PCI DSS include:\n",
+      "\n",
+      "1. Scope: Penetration testing involves simulated attacks on the network and systems to identify vulnerabilities that could be exploited, while vulnerability assessments focus on identifying and classifying existing vulnerabilities in the system.\n",
+      "\n",
+      "2. Methodology: Penetration testing uses a more aggressive approach, attempting to exploit vulnerabilities and gain access to sensitive data or systems, whereas vulnerability assessments use a more passive approach, scanning for known vulnerabilities without actually exploiting them.\n",
+      "\n",
+      "3. Frequency: Penetration testing is typically conducted annually, while vulnerability assessments may be conducted more frequently, depending on the organization's risk profile and compliance requirements.\n",
+      "\n",
+      "4. Qualified individuals: Penetration testing requires a qualified individual to perform the test, while vulnerability assessments can be performed by internal staff or external third parties, as long as they have the necessary skills and knowledge.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 7.499s\n",
+      "Total time used: 20.195 s\n",
+      "Number of tokens generated: 652\n",
+      "Average generation speed: 32.286 tokens/s\n",
+      "CPU times: user 380 ms, sys: 213 ms, total: 593 ms\n",
+      "Wall time: 41.7 s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "os.environ[\"HF_RP\"] = \"1.10\"\n",
+    "!python qa_chain_test.py"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "db48dcc4",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading env vars from: /home/inflaton/code/chat-with-orca-2/.env\n",
+      "Running on:  Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.35\n",
+      "MPS is NOT available\n",
+      "CUDA is  available\n",
+      "MPS not available because the current PyTorch install was not built with MPS enabled.\n",
+      "CUDA is available, we have found  1  GPU(s)\n",
+      "NVIDIA GeForce RTX 4090\n",
+      "CUDA version: 12.1\n",
+      "hf_embeddings_device_type: cuda:0\n",
+      "hf_pipeline_device_type: cuda:0\n",
+      "load INSTRUCTOR_Transformer\n",
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n",
+      "max_seq_length  512\n",
+      "Completed in 0.456s\n",
+      "Load index from ./data/faiss_1024_512/ with FAISS\n",
+      "Completed in 0.016s\n",
+      "initializing LLM: huggingface\n",
+      "  hf_pipeline_device_type: cuda:0\n",
+      "     load_quantized_model: None\n",
+      "              torch_dtype: torch.bfloat16\n",
+      "                 n_threds: 4\n",
+      "            loading model: microsoft/Orca-2-7b\n",
+      "            HF auth token: None\n",
+      "       repetition_penalty: 1.15\n",
+      "Loading checkpoint shards: 100%|██████████████████| 3/3 [00:01<00:00,  1.59it/s]\n",
+      "initialization complete\n",
+      "Completed in 6.345s\n",
+      "\n",
+      "Question: What's PCI DSS?\n",
+      "{'question': \"What's PCI DSS?\", 'chat_history': []}\n",
+      "PCI DSS stands for Payment Card Industry Data Security Standard. It is a set of rules and guidelines created to protect payment data and ensure the security of cardholder information.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 3.256s\n",
+      "\n",
+      "Question: Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?\n",
+      "{'question': 'Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a set of rules and guidelines created to protect payment data and ensure the security of cardholder information.')]}\n",
+      "How has the latest version of PCI DSS, version 4.0, differed from the previous version, 3.2.1?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "The latest version of PCI DSS, version 4.0, differs from the previous version, 3.2.1, by including updates to align with the changes from PCI DSS v3.2.1 to PCI DSS v4.0.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 2.222s\n",
+      "\n",
+      "Question: new requirements for vulnerability assessments\n",
+      "{'question': 'new requirements for vulnerability assessments', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a set of rules and guidelines created to protect payment data and ensure the security of cardholder information.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'The latest version of PCI DSS, version 4.0, differs from the previous version, 3.2.1, by including updates to align with the changes from PCI DSS v3.2.1 to PCI DSS v4.0.')]}\n",
+      "What are some new requirements for vulnerability assessments introduced in PCI DSS version 4.0?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Some new requirements for vulnerability assessments introduced in PCI DSS version 4.0 include:\n",
+      "\n",
+      "- Performing a targeted risk analysis to determine the frequency of log reviews for all other system components (Requirement 10.4.2.1)\n",
+      "- Ensuring failures of critical security control systems are detected, alerted, and addressed promptly (Requirement 10.7.2), and responded to promptly (Requirement 10.7.3)\n",
+      "- Documenting, assigning, and understanding roles and responsibilities for performing activities in Requirement 11 (Requirement 11.1.2)\n",
+      "- Managing all applicable vulnerabilities not ranked as high-risk or critical (Requirement 11.3.1.1)\n",
+      "- Performing internal vulnerability scans via authenticated scanning (Requirement 11.3.1.2)\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 5.732s\n",
+      "\n",
+      "Question: more on penetration testing\n",
+      "{'question': 'more on penetration testing', 'chat_history': [(\"What's PCI DSS?\", 'PCI DSS stands for Payment Card Industry Data Security Standard. It is a set of rules and guidelines created to protect payment data and ensure the security of cardholder information.'), ('Can you summarize the changes made from PCI DSS version 3.2.1 to version 4.0?', 'The latest version of PCI DSS, version 4.0, differs from the previous version, 3.2.1, by including updates to align with the changes from PCI DSS v3.2.1 to PCI DSS v4.0.'), ('new requirements for vulnerability assessments', 'Some new requirements for vulnerability assessments introduced in PCI DSS version 4.0 include:\\n\\n- Performing a targeted risk analysis to determine the frequency of log reviews for all other system components (Requirement 10.4.2.1)\\n- Ensuring failures of critical security control systems are detected, alerted, and addressed promptly (Requirement 10.7.2), and responded to promptly (Requirement 10.7.3)\\n- Documenting, assigning, and understanding roles and responsibilities for performing activities in Requirement 11 (Requirement 11.1.2)\\n- Managing all applicable vulnerabilities not ranked as high-risk or critical (Requirement 11.3.1.1)\\n- Performing internal vulnerability scans via authenticated scanning (Requirement 11.3.1.2)')]}\n",
+      "What are some best practices for conducting effective penetration tests?\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Some best practices for conducting effective penetration tests include performing the testing according to a defined methodology, ensuring the tester has organizational independence, validating segmentation and scope reduction controls, testing both inside and outside the network, considering the latest threats and vulnerabilities, and documenting the approach to addressing risks identified during the testing.\n",
+      "\n",
+      "\n",
+      "\n",
+      "<on_llm_end>\n",
+      "\n",
+      "\n",
+      "Completed in 2.461s\n",
+      "Total time used: 13.672 s\n",
+      "Number of tokens generated: 454\n",
+      "Average generation speed: 33.208 tokens/s\n",
+      "CPU times: user 322 ms, sys: 44.4 ms, total: 366 ms\n",
+      "Wall time: 23.7 s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "os.environ[\"HF_RP\"] = \"1.15\"\n",
+    "!python qa_chain_test.py"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

qa_chain_test.py

@@ -0,0 +1,88 @@
+import os
+import sys
+from timeit import default_timer as timer
+
+from langchain.callbacks.base import BaseCallbackHandler
+from langchain.schema import LLMResult
+
+from app_modules.init import app_init
+from app_modules.utils import print_llm_response
+
+llm_loader, qa_chain = app_init()
+
+
+class MyCustomHandler(BaseCallbackHandler):
+    def __init__(self):
+        self.reset()
+
+    def reset(self):
+        self.texts = []
+
+    def get_standalone_question(self) -> str:
+        return self.texts[0].strip() if len(self.texts) > 0 else None
+
+    def on_llm_end(self, response: LLMResult, **kwargs) -> None:
+        """Run when chain ends running."""
+        print("\n<on_llm_end>")
+        # print(response)
+        self.texts.append(response.generations[0][0].text)
+
+
+chatting = len(sys.argv) > 1 and sys.argv[1] == "chat"
+questions_file_path = os.environ.get("QUESTIONS_FILE_PATH")
+chat_history_enabled = os.environ.get("CHAT_HISTORY_ENABLED") or "true"
+
+custom_handler = MyCustomHandler()
+
+# Chatbot loop
+chat_history = []
+
+# Open the file for reading
+file = open(questions_file_path, "r")
+
+# Read the contents of the file into a list of strings
+questions = file.readlines()
+for i in range(len(questions)):
+    questions[i] = questions[i].strip()
+
+# Close the file
+file.close()
+
+if __name__ == "__main__":
+    questions.append("exit")
+
+    chat_start = timer()
+
+    while True:
+        if chatting:
+            query = input("Please enter your question: ")
+        else:
+            query = questions.pop(0)
+
+        query = query.strip()
+        if query.lower() == "exit":
+            break
+
+        print("\nQuestion: " + query)
+        custom_handler.reset()
+
+        start = timer()
+        result = qa_chain.call_chain(
+            {"question": query, "chat_history": chat_history},
+            custom_handler,
+            None,
+            True,
+        )
+        end = timer()
+        print(f"Completed in {end - start:.3f}s")
+
+        if chat_history_enabled == "true":
+            chat_history.append((query, result["answer"]))
+
+    chat_end = timer()
+    total_time = chat_end - chat_start
+    print(f"Total time used: {total_time:.3f} s")
+    print(f"Number of tokens generated: {llm_loader.streamer.total_tokens}")
+    print(
+        f"Average generation speed: {llm_loader.streamer.total_tokens / total_time:.3f} tokens/s"
+    )

requirements-mac.txt

@@ -0,0 +1,129 @@
+accelerate==0.26.1
+aiofiles==23.2.1
+aiohttp==3.9.1
+aiosignal==1.3.1
+altair==5.2.0
+annotated-types==0.6.0
+anyio==4.2.0
+attrs==23.2.0
+black==24.1.0
+certifi==2023.11.17
+charset-normalizer==3.3.2
+click==8.1.7
+colorama==0.4.6
+contourpy==1.2.0
+cycler==0.12.1
+dataclasses-json==0.6.3
+faiss-cpu==1.7.4
+fastapi==0.109.0
+ffmpy==0.3.1
+filelock==3.13.1
+fonttools==4.47.2
+frozenlist==1.4.1
+fsspec==2023.12.2
+gradio==4.16.0
+gradio_client==0.8.1
+greenlet==3.0.3
+h11==0.14.0
+httpcore==1.0.2
+httpx==0.26.0
+huggingface-hub==0.20.3
+idna==3.6
+importlib-resources==6.1.1
+InstructorEmbedding==1.0.1
+isort==5.13.2
+Jinja2==3.1.3
+joblib==1.3.2
+jsonpatch==1.33
+jsonpointer==2.4
+jsonschema==4.21.1
+jsonschema-specifications==2023.12.1
+kiwisolver==1.4.5
+langchain==0.1.4
+langchain-community==0.0.16
+langchain-openai==0.0.5
+langchain-core==0.1.16
+langsmith==0.0.83
+markdown-it-py==3.0.0
+MarkupSafe==2.1.4
+marshmallow==3.20.2
+matplotlib==3.8.2
+mdurl==0.1.2
+mpmath==1.3.0
+multidict==6.0.4
+mypy-extensions==1.0.0
+networkx==3.2.1
+nltk==3.8.1
+numpy==1.26.3
+# nvidia-cublas-cu12==12.1.3.1
+# nvidia-cuda-cupti-cu12==12.1.105
+# nvidia-cuda-nvrtc-cu12==12.1.105
+# nvidia-cuda-runtime-cu12==12.1.105
+# nvidia-cudnn-cu12==8.9.2.26
+# nvidia-cufft-cu12==11.0.2.54
+# nvidia-curand-cu12==10.3.2.106
+# nvidia-cusolver-cu12==11.4.5.107
+# nvidia-cusparse-cu12==12.1.0.106
+# nvidia-nccl-cu12==2.18.1
+# nvidia-nvjitlink-cu12==12.3.101
+# nvidia-nvtx-cu12==12.1.105
+orjson==3.9.12
+packaging==23.2
+pandas==2.2.0
+pathspec==0.12.1
+peft @ git+https://github.com/huggingface/peft.git@1c1c7fdaa6e6abaa53939b865dee1eded82ad032
+pillow==10.2.0
+platformdirs==4.1.0
+protobuf==4.25.2
+psutil==5.9.8
+pydantic==2.5.3
+pydantic_core==2.14.6
+pydub==0.25.1
+Pygments==2.17.2
+pyparsing==3.1.1
+python-dateutil==2.8.2
+python-dotenv==1.0.1
+python-multipart==0.0.6
+pytz==2023.3.post1
+PyYAML==6.0.1
+referencing==0.32.1
+regex==2023.12.25
+requests==2.31.0
+rich==13.7.0
+rpds-py==0.17.1
+ruff==0.1.14
+safetensors==0.4.2
+scikit-learn==1.4.0
+scipy==1.12.0
+semantic-version==2.10.0
+sentence-transformers==2.2.2
+sentencepiece==0.1.99
+shellingham==1.5.4
+six==1.16.0
+sniffio==1.3.0
+SQLAlchemy==2.0.25
+starlette==0.35.1
+sympy==1.12
+tenacity==8.2.3
+threadpoolctl==3.2.0
+tokenizers==0.15.1
+tomlkit==0.12.0
+toolz==0.12.1
+torch==2.1.2
+torchvision==0.16.2
+tqdm==4.66.1
+transformers @ git+https://github.com/huggingface/transformers.git@de13a951b38b85195984164819f1ab05fe508677
+# triton==2.1.0
+typer==0.9.0
+typing-inspect==0.9.0
+typing_extensions==4.9.0
+tzdata==2023.4
+urllib3==2.1.0
+uvicorn==0.27.0
+websockets==11.0.3
+yarl==1.9.4
+einops==0.7.0
+Pyarrow==15.0.0
+openpyxl==3.1.2
+text_generation==0.6.1
+tabulate==0.9.0

requirements.txt

@@ -0,0 +1,129 @@
+accelerate==0.26.1
+aiofiles==23.2.1
+aiohttp==3.9.1
+aiosignal==1.3.1
+altair==5.2.0
+annotated-types==0.6.0
+anyio==4.2.0
+attrs==23.2.0
+black==24.1.0
+certifi==2023.11.17
+charset-normalizer==3.3.2
+click==8.1.7
+colorama==0.4.6
+contourpy==1.2.0
+cycler==0.12.1
+dataclasses-json==0.6.3
+faiss-cpu==1.7.4
+fastapi==0.109.0
+ffmpy==0.3.1
+filelock==3.13.1
+fonttools==4.47.2
+frozenlist==1.4.1
+fsspec==2023.12.2
+gradio==4.16.0
+gradio_client==0.8.1
+greenlet==3.0.3
+h11==0.14.0
+httpcore==1.0.2
+httpx==0.26.0
+huggingface-hub==0.20.3
+idna==3.6
+importlib-resources==6.1.1
+InstructorEmbedding==1.0.1
+isort==5.13.2
+Jinja2==3.1.3
+joblib==1.3.2
+jsonpatch==1.33
+jsonpointer==2.4
+jsonschema==4.21.1
+jsonschema-specifications==2023.12.1
+kiwisolver==1.4.5
+langchain==0.1.4
+langchain-community==0.0.16
+langchain-openai==0.0.5
+langchain-core==0.1.16
+langsmith==0.0.83
+markdown-it-py==3.0.0
+MarkupSafe==2.1.4
+marshmallow==3.20.2
+matplotlib==3.8.2
+mdurl==0.1.2
+mpmath==1.3.0
+multidict==6.0.4
+mypy-extensions==1.0.0
+networkx==3.2.1
+nltk==3.8.1
+numpy==1.26.3
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.18.1
+nvidia-nvjitlink-cu12==12.3.101
+nvidia-nvtx-cu12==12.1.105
+orjson==3.9.12
+packaging==23.2
+pandas==2.2.0
+pathspec==0.12.1
+peft @ git+https://github.com/huggingface/peft.git@1c1c7fdaa6e6abaa53939b865dee1eded82ad032
+pillow==10.2.0
+platformdirs==4.1.0
+protobuf==4.25.2
+psutil==5.9.8
+pydantic==2.5.3
+pydantic_core==2.14.6
+pydub==0.25.1
+Pygments==2.17.2
+pyparsing==3.1.1
+python-dateutil==2.8.2
+python-dotenv==1.0.1
+python-multipart==0.0.6
+pytz==2023.3.post1
+PyYAML==6.0.1
+referencing==0.32.1
+regex==2023.12.25
+requests==2.31.0
+rich==13.7.0
+rpds-py==0.17.1
+ruff==0.1.14
+safetensors==0.4.2
+scikit-learn==1.4.0
+scipy==1.12.0
+semantic-version==2.10.0
+sentence-transformers==2.2.2
+sentencepiece==0.1.99
+shellingham==1.5.4
+six==1.16.0
+sniffio==1.3.0
+SQLAlchemy==2.0.25
+starlette==0.35.1
+sympy==1.12
+tenacity==8.2.3
+threadpoolctl==3.2.0
+tokenizers==0.15.1
+tomlkit==0.12.0
+toolz==0.12.1
+torch==2.1.2
+torchvision==0.16.2
+tqdm==4.66.1
+transformers @ git+https://github.com/huggingface/transformers.git@de13a951b38b85195984164819f1ab05fe508677
+triton==2.1.0
+typer==0.9.0
+typing-inspect==0.9.0
+typing_extensions==4.9.0
+tzdata==2023.4
+urllib3==2.1.0
+uvicorn==0.27.0
+websockets==11.0.3
+yarl==1.9.4
+einops==0.7.0
+Pyarrow==15.0.0
+openpyxl==3.1.2
+text_generation==0.6.1
+tabulate==0.9.0

results/1_process-raw-data.ipynb

@@ -0,0 +1,1573 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "workding dir: /home/inflaton/code/chat-with-orca-2\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import sys\n",
+    "from pathlib import Path\n",
+    "\n",
+    "workding_dir = str(Path.cwd().parent)\n",
+    "os.chdir(workding_dir)\n",
+    "sys.path.append(workding_dir)\n",
+    "print(\"workding dir:\", workding_dir)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading env vars from: /home/inflaton/code/chat-with-orca-2/.env\n",
+      "Running on:  Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.35\n",
+      "MPS is NOT available\n",
+      "CUDA is  available\n",
+      "MPS not available because the current PyTorch install was not built with MPS enabled.\n",
+      "CUDA is available, we have found  1  GPU(s)\n",
+      "NVIDIA GeForce RTX 4090\n",
+      "CUDA version: 12.1\n",
+      "hf_embeddings_device_type: cuda:0\n",
+      "hf_pipeline_device_type: cuda:0\n",
+      "load INSTRUCTOR_Transformer\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "max_seq_length  512\n",
+      "Completed in 0.464s\n",
+      "Load index from ./data/faiss_1024_512/ with FAISS\n",
+      "Completed in 0.013s\n",
+      "initializing LLM: openai\n",
+      "  hf_pipeline_device_type: cuda:0\n",
+      "     load_quantized_model: None\n",
+      "              torch_dtype: torch.bfloat16\n",
+      "                 n_threds: 4\n",
+      "              using model: gpt-3.5-turbo\n",
+      "initialization complete\n",
+      "Completed in 0.024s\n",
+      "CPU times: user 5.59 s, sys: 5.55 s, total: 11.1 s\n",
+      "Wall time: 3.47 s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "os.environ[\"LLM_MODEL_TYPE\"] = \"openai\"\n",
+    "from eval_models import *"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading:  ./notebook/nvidia-4090-orca-2-7b.ipynb\n",
+      "processing repetition_penalty: 1.05\n",
+      "processing repetition_penalty: 1.10\n",
+      "processing repetition_penalty: 1.15\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating: 100%|██████████| 8/8 [00:52<00:00,  6.53s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "# Ragas scores: {'faithfulness': 0.8304, 'answer_relevancy': 0.9783, 'overall_score': 0.8983}\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating: 100%|██████████| 8/8 [00:31<00:00,  3.97s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "# Ragas scores: {'faithfulness': 0.7500, 'answer_relevancy': 0.9748, 'overall_score': 0.8478}\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating: 100%|██████████| 8/8 [00:29<00:00,  3.65s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "# Ragas scores: {'faithfulness': 1.0000, 'answer_relevancy': 0.9733, 'overall_score': 0.9865}\n",
+      "\n",
+      "loading:  ./notebook/nvidia-4090-llama-2-7b.ipynb\n",
+      "processing repetition_penalty: 1.05\n",
+      "processing repetition_penalty: 1.10\n",
+      "processing repetition_penalty: 1.15\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating: 100%|██████████| 8/8 [00:35<00:00,  4.39s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "# Ragas scores: {'faithfulness': 0.8750, 'answer_relevancy': 0.7151, 'overall_score': 0.7870}\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating: 100%|██████████| 8/8 [00:40<00:00,  5.02s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "# Ragas scores: {'faithfulness': 0.8796, 'answer_relevancy': 0.7313, 'overall_score': 0.7986}\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating: 100%|██████████| 8/8 [00:53<00:00,  6.63s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "# Ragas scores: {'faithfulness': 1.0000, 'answer_relevancy': 0.7112, 'overall_score': 0.8312}\n",
+      "\n",
+      "loading:  ./notebook/nvidia-4090-orca-2-13b.ipynb\n",
+      "processing repetition_penalty: 1.05\n",
+      "processing repetition_penalty: 1.10\n",
+      "processing repetition_penalty: 1.15\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating: 100%|██████████| 8/8 [00:44<00:00,  5.54s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "# Ragas scores: {'faithfulness': 1.0000, 'answer_relevancy': 0.9876, 'overall_score': 0.9938}\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating: 100%|██████████| 8/8 [00:47<00:00,  5.99s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "# Ragas scores: {'faithfulness': 1.0000, 'answer_relevancy': 0.9608, 'overall_score': 0.9800}\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating: 100%|██████████| 8/8 [02:41<00:00, 20.19s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "# Ragas scores: {'faithfulness': 0.9500, 'answer_relevancy': 0.9611, 'overall_score': 0.9555}\n",
+      "\n",
+      "loading:  ./notebook/nvidia-4090-llama-2-13b.ipynb\n",
+      "processing repetition_penalty: 1.05\n",
+      "processing repetition_penalty: 1.10\n",
+      "processing repetition_penalty: 1.15\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating: 100%|██████████| 8/8 [00:33<00:00,  4.20s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "# Ragas scores: {'faithfulness': 0.9000, 'answer_relevancy': 0.9624, 'overall_score': 0.9302}\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating: 100%|██████████| 8/8 [00:41<00:00,  5.17s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "# Ragas scores: {'faithfulness': 0.8750, 'answer_relevancy': 0.9673, 'overall_score': 0.9188}\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating: 100%|██████████| 8/8 [00:50<00:00,  6.26s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "# Ragas scores: {'faithfulness': 0.9444, 'answer_relevancy': 0.9646, 'overall_score': 0.9544}\n",
+      "\n",
+      "loading:  ./notebook/nvidia-4090-openai.ipynb\n",
+      "processing repetition_penalty: 1.05\n",
+      "processing repetition_penalty: 1.10\n",
+      "processing repetition_penalty: 1.15\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating: 100%|██████████| 8/8 [02:41<00:00, 20.17s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "# Ragas scores: {'faithfulness': 0.9583, 'answer_relevancy': 0.4836, 'overall_score': 0.6428}\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating: 100%|██████████| 8/8 [00:36<00:00,  4.54s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "# Ragas scores: {'faithfulness': 0.8373, 'answer_relevancy': 0.9527, 'overall_score': 0.8913}\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating: 100%|██████████| 8/8 [03:09<00:00, 23.67s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "# Ragas scores: {'faithfulness': 1.0000, 'answer_relevancy': 0.7019, 'overall_score': 0.8248}\n",
+      "\n",
+      "CPU times: user 5.91 s, sys: 930 ms, total: 6.84 s\n",
+      "Wall time: 17min 18s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "model_names = [\"orca-2-7b\", \"llama-2-7b\", \"orca-2-13b\", \"llama-2-13b\", \"openai\"]\n",
+    "\n",
+    "perf_pd, raw_pd = evaluate_models(model_names)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>model_name</th>\n",
+       "      <th>repetition_penalty</th>\n",
+       "      <th>faithfulness</th>\n",
+       "      <th>answer_relevancy</th>\n",
+       "      <th>overall_score</th>\n",
+       "      <th>total_time_used</th>\n",
+       "      <th>num_tokens_generated</th>\n",
+       "      <th>token_per_second</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>orca-2-7b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>0.830357</td>\n",
+       "      <td>0.978324</td>\n",
+       "      <td>0.898288</td>\n",
+       "      <td>46.121</td>\n",
+       "      <td>536</td>\n",
+       "      <td>11.622</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>orca-2-7b</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>0.750000</td>\n",
+       "      <td>0.974817</td>\n",
+       "      <td>0.847757</td>\n",
+       "      <td>20.195</td>\n",
+       "      <td>652</td>\n",
+       "      <td>32.286</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>orca-2-7b</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>0.973278</td>\n",
+       "      <td>0.986458</td>\n",
+       "      <td>13.672</td>\n",
+       "      <td>454</td>\n",
+       "      <td>33.208</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>llama-2-7b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>0.875000</td>\n",
+       "      <td>0.715099</td>\n",
+       "      <td>0.787010</td>\n",
+       "      <td>19.468</td>\n",
+       "      <td>679</td>\n",
+       "      <td>34.878</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>llama-2-7b</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>0.879630</td>\n",
+       "      <td>0.731304</td>\n",
+       "      <td>0.798638</td>\n",
+       "      <td>21.670</td>\n",
+       "      <td>759</td>\n",
+       "      <td>35.026</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>llama-2-7b</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>0.711172</td>\n",
+       "      <td>0.831210</td>\n",
+       "      <td>22.604</td>\n",
+       "      <td>803</td>\n",
+       "      <td>35.524</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>orca-2-13b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>0.987592</td>\n",
+       "      <td>0.993757</td>\n",
+       "      <td>397.548</td>\n",
+       "      <td>641</td>\n",
+       "      <td>1.612</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>orca-2-13b</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>0.960806</td>\n",
+       "      <td>0.980011</td>\n",
+       "      <td>272.891</td>\n",
+       "      <td>478</td>\n",
+       "      <td>1.752</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>orca-2-13b</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>0.950000</td>\n",
+       "      <td>0.961115</td>\n",
+       "      <td>0.955525</td>\n",
+       "      <td>291.610</td>\n",
+       "      <td>514</td>\n",
+       "      <td>1.763</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>llama-2-13b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>0.900000</td>\n",
+       "      <td>0.962428</td>\n",
+       "      <td>0.930168</td>\n",
+       "      <td>369.084</td>\n",
+       "      <td>677</td>\n",
+       "      <td>1.834</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>10</th>\n",
+       "      <td>llama-2-13b</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>0.875000</td>\n",
+       "      <td>0.967267</td>\n",
+       "      <td>0.918823</td>\n",
+       "      <td>505.816</td>\n",
+       "      <td>881</td>\n",
+       "      <td>1.742</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>11</th>\n",
+       "      <td>llama-2-13b</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>0.944444</td>\n",
+       "      <td>0.964647</td>\n",
+       "      <td>0.954439</td>\n",
+       "      <td>435.429</td>\n",
+       "      <td>777</td>\n",
+       "      <td>1.784</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>12</th>\n",
+       "      <td>gpt-3.5-turbo</td>\n",
+       "      <td></td>\n",
+       "      <td>0.958333</td>\n",
+       "      <td>0.483574</td>\n",
+       "      <td>0.642795</td>\n",
+       "      <td>13.232</td>\n",
+       "      <td>425</td>\n",
+       "      <td>32.119</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13</th>\n",
+       "      <td>gpt-3.5-turbo-instruct</td>\n",
+       "      <td></td>\n",
+       "      <td>0.837302</td>\n",
+       "      <td>0.952711</td>\n",
+       "      <td>0.891286</td>\n",
+       "      <td>10.360</td>\n",
+       "      <td>411</td>\n",
+       "      <td>39.673</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>14</th>\n",
+       "      <td>gpt-4</td>\n",
+       "      <td></td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>0.701869</td>\n",
+       "      <td>0.824822</td>\n",
+       "      <td>42.257</td>\n",
+       "      <td>670</td>\n",
+       "      <td>15.855</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                model_name repetition_penalty  faithfulness  answer_relevancy  \\\n",
+       "0                orca-2-7b               1.05      0.830357          0.978324   \n",
+       "1                orca-2-7b               1.10      0.750000          0.974817   \n",
+       "2                orca-2-7b               1.15      1.000000          0.973278   \n",
+       "3               llama-2-7b               1.05      0.875000          0.715099   \n",
+       "4               llama-2-7b               1.10      0.879630          0.731304   \n",
+       "5               llama-2-7b               1.15      1.000000          0.711172   \n",
+       "6               orca-2-13b               1.05      1.000000          0.987592   \n",
+       "7               orca-2-13b               1.10      1.000000          0.960806   \n",
+       "8               orca-2-13b               1.15      0.950000          0.961115   \n",
+       "9              llama-2-13b               1.05      0.900000          0.962428   \n",
+       "10             llama-2-13b               1.10      0.875000          0.967267   \n",
+       "11             llama-2-13b               1.15      0.944444          0.964647   \n",
+       "12           gpt-3.5-turbo                         0.958333          0.483574   \n",
+       "13  gpt-3.5-turbo-instruct                         0.837302          0.952711   \n",
+       "14                   gpt-4                         1.000000          0.701869   \n",
+       "\n",
+       "    overall_score  total_time_used  num_tokens_generated  token_per_second  \n",
+       "0        0.898288           46.121                   536            11.622  \n",
+       "1        0.847757           20.195                   652            32.286  \n",
+       "2        0.986458           13.672                   454            33.208  \n",
+       "3        0.787010           19.468                   679            34.878  \n",
+       "4        0.798638           21.670                   759            35.026  \n",
+       "5        0.831210           22.604                   803            35.524  \n",
+       "6        0.993757          397.548                   641             1.612  \n",
+       "7        0.980011          272.891                   478             1.752  \n",
+       "8        0.955525          291.610                   514             1.763  \n",
+       "9        0.930168          369.084                   677             1.834  \n",
+       "10       0.918823          505.816                   881             1.742  \n",
+       "11       0.954439          435.429                   777             1.784  \n",
+       "12       0.642795           13.232                   425            32.119  \n",
+       "13       0.891286           10.360                   411            39.673  \n",
+       "14       0.824822           42.257                   670            15.855  "
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "perf_pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>model_name</th>\n",
+       "      <th>repetition_penalty</th>\n",
+       "      <th>user_question</th>\n",
+       "      <th>standalone_question</th>\n",
+       "      <th>contexts</th>\n",
+       "      <th>answer</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>orca-2-7b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>What's PCI DSS?</td>\n",
+       "      <td></td>\n",
+       "      <td>[PCI Data Security Standard\\nPCI DSS provides ...</td>\n",
+       "      <td>PCI DSS stands for Payment Card Industry Data ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>orca-2-7b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>Can you summarize the changes made from PCI DS...</td>\n",
+       "      <td>What are the main differences between PCI DSS ...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>The main differences between PCI DSS version 3...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>orca-2-7b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>new requirements for vulnerability assessments</td>\n",
+       "      <td>What are the new requirements for vulnerabilit...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>The new requirements for vulnerability assessm...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>orca-2-7b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>more on penetration testing</td>\n",
+       "      <td>What are the new requirements for penetration ...</td>\n",
+       "      <td>[PCI DSS v4.0 ROC Template r1  December 2022  ...</td>\n",
+       "      <td>The new requirements for penetration testing i...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>orca-2-7b</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>What's PCI DSS?</td>\n",
+       "      <td></td>\n",
+       "      <td>[PCI Data Security Standard\\nPCI DSS provides ...</td>\n",
+       "      <td>PCI DSS stands for Payment Card Industry Data ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>orca-2-7b</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>Can you summarize the changes made from PCI DS...</td>\n",
+       "      <td>What are the main differences between PCI DSS ...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>The main differences between PCI DSS versions ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>orca-2-7b</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>new requirements for vulnerability assessments</td>\n",
+       "      <td>What are the new requirements for vulnerabilit...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>There are several new requirements related to ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>orca-2-7b</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>more on penetration testing</td>\n",
+       "      <td>What are some key differences between penetrat...</td>\n",
+       "      <td>[PCI DSS v4.0 ROC Template r1  December 2022  ...</td>\n",
+       "      <td>Some key differences between penetration testi...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>orca-2-7b</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>What's PCI DSS?</td>\n",
+       "      <td></td>\n",
+       "      <td>[PCI Data Security Standard\\nPCI DSS provides ...</td>\n",
+       "      <td>PCI DSS stands for Payment Card Industry Data ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>orca-2-7b</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>Can you summarize the changes made from PCI DS...</td>\n",
+       "      <td>How has the latest version of PCI DSS, version...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>The latest version of PCI DSS, version 4.0, di...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>10</th>\n",
+       "      <td>orca-2-7b</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>new requirements for vulnerability assessments</td>\n",
+       "      <td>What are some new requirements for vulnerabili...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>Some new requirements for vulnerability assess...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>11</th>\n",
+       "      <td>orca-2-7b</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>more on penetration testing</td>\n",
+       "      <td>What are some best practices for conducting ef...</td>\n",
+       "      <td>[methodology  examined for this testing \\nproc...</td>\n",
+       "      <td>Some best practices for conducting effective p...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>12</th>\n",
+       "      <td>llama-2-7b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>What's PCI DSS?</td>\n",
+       "      <td></td>\n",
+       "      <td>[PCI Data Security Standard\\nPCI DSS provides ...</td>\n",
+       "      <td>PCI DSS stands for Payment Card Industry Data ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13</th>\n",
+       "      <td>llama-2-7b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>Can you summarize the changes made from PCI DS...</td>\n",
+       "      <td>What are the key changes made in PCI DSS versi...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>Based on the provided document, the key chang...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>14</th>\n",
+       "      <td>llama-2-7b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>new requirements for vulnerability assessments</td>\n",
+       "      <td>What are the new requirements for vulnerabilit...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>According to the provided summary of changes,...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>15</th>\n",
+       "      <td>llama-2-7b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>more on penetration testing</td>\n",
+       "      <td>What are the new requirements for penetration ...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>The new requirement for penetration testing i...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>16</th>\n",
+       "      <td>llama-2-7b</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>What's PCI DSS?</td>\n",
+       "      <td></td>\n",
+       "      <td>[PCI Data Security Standard\\nPCI DSS provides ...</td>\n",
+       "      <td>PCI DSS stands for Payment Card Industry Data ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>17</th>\n",
+       "      <td>llama-2-7b</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>Can you summarize the changes made from PCI DS...</td>\n",
+       "      <td>What are the key changes made in PCI DSS versi...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>The key changes made in PCI DSS version 4.0 co...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>18</th>\n",
+       "      <td>llama-2-7b</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>new requirements for vulnerability assessments</td>\n",
+       "      <td>What are the new requirements for vulnerabilit...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>The new requirement for vulnerability assessme...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>19</th>\n",
+       "      <td>llama-2-7b</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>more on penetration testing</td>\n",
+       "      <td>What are the new requirements for penetration ...</td>\n",
+       "      <td>[PCI DSS v4.0 ROC Template r1  December 2022  ...</td>\n",
+       "      <td>The new requirements for penetration testing i...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>20</th>\n",
+       "      <td>llama-2-7b</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>What's PCI DSS?</td>\n",
+       "      <td></td>\n",
+       "      <td>[PCI Data Security Standard\\nPCI DSS provides ...</td>\n",
+       "      <td>According to the given quick reference guide, ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>21</th>\n",
+       "      <td>llama-2-7b</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>Can you summarize the changes made from PCI DS...</td>\n",
+       "      <td>What are the key changes between PCI DSS versi...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>The document highlights several significant mo...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>22</th>\n",
+       "      <td>llama-2-7b</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>new requirements for vulnerability assessments</td>\n",
+       "      <td>What are some of the new requirements for vuln...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>According to the provided documents, one of th...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>23</th>\n",
+       "      <td>llama-2-7b</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>more on penetration testing</td>\n",
+       "      <td>Could you explain what penetration testing ent...</td>\n",
+       "      <td>[PCI DSS v4.0 ROC Template r1  December 2022  ...</td>\n",
+       "      <td>Sure! Penetration testing is a crucial compone...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>24</th>\n",
+       "      <td>orca-2-13b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>What's PCI DSS?</td>\n",
+       "      <td></td>\n",
+       "      <td>[PCI Data Security Standard\\nPCI DSS provides ...</td>\n",
+       "      <td>PCI DSS is a global standard that provides a b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>25</th>\n",
+       "      <td>orca-2-13b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>Can you summarize the changes made from PCI DS...</td>\n",
+       "      <td>¿Puedes resumir los cambios realizados desde l...</td>\n",
+       "      <td>[April 2015  PCI DSS 3.1, \\nRevision  1.0 Revi...</td>\n",
+       "      <td>Sí, puedo resumir los cambios realizados desde...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>26</th>\n",
+       "      <td>orca-2-13b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>new requirements for vulnerability assessments</td>\n",
+       "      <td>¿Cuáles son las nuevas requisitos para las eva...</td>\n",
+       "      <td>[The Prioritized Approach to Pursue PCI DSS Co...</td>\n",
+       "      <td>Las nuevas requisitos para las evaluaciones de...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>27</th>\n",
+       "      <td>orca-2-13b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>more on penetration testing</td>\n",
+       "      <td>¿Puedes dar más detalles sobre las prácticas d...</td>\n",
+       "      <td>[The Prioritized Approach to Pursue PCI DSS Co...</td>\n",
+       "      <td>Sí, puedo dar más detalles sobre las prácticas...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>28</th>\n",
+       "      <td>orca-2-13b</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>What's PCI DSS?</td>\n",
+       "      <td></td>\n",
+       "      <td>[PCI Data Security Standard\\nPCI DSS provides ...</td>\n",
+       "      <td>PCI DSS is a set of requirements designed to p...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>29</th>\n",
+       "      <td>orca-2-13b</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>Can you summarize the changes made from PCI DS...</td>\n",
+       "      <td>¿Puedes resumir los cambios realizados desde l...</td>\n",
+       "      <td>[April 2015  PCI DSS 3.1, \\nRevision  1.0 Revi...</td>\n",
+       "      <td>Sí, puedo resumir los cambios realizados desde...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>30</th>\n",
+       "      <td>orca-2-13b</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>new requirements for vulnerability assessments</td>\n",
+       "      <td>¿Cuáles son las nuevas exigencias para las eva...</td>\n",
+       "      <td>[The Prioritized Approach to Pursue PCI DSS Co...</td>\n",
+       "      <td>Las nuevas exigencias para las evaluaciones de...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>31</th>\n",
+       "      <td>orca-2-13b</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>more on penetration testing</td>\n",
+       "      <td>¿Puedo obtener más información sobre las prueb...</td>\n",
+       "      <td>[The Prioritized Approach to Pursue PCI DSS Co...</td>\n",
+       "      <td>Yes, you can get more information about penetr...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>32</th>\n",
+       "      <td>orca-2-13b</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>What's PCI DSS?</td>\n",
+       "      <td></td>\n",
+       "      <td>[PCI Data Security Standard\\nPCI DSS provides ...</td>\n",
+       "      <td>PCI DSS is a set of requirements designed to p...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>33</th>\n",
+       "      <td>orca-2-13b</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>Can you summarize the changes made from PCI DS...</td>\n",
+       "      <td>¿Puedes resumir los cambios realizados desde l...</td>\n",
+       "      <td>[April 2015  PCI DSS 3.1, \\nRevision  1.0 Revi...</td>\n",
+       "      <td>Sí, puedo resumir los cambios realizados desde...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>34</th>\n",
+       "      <td>orca-2-13b</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>new requirements for vulnerability assessments</td>\n",
+       "      <td>¿Cuáles son las nuevas recomendaciones para la...</td>\n",
+       "      <td>[The Prioritized Approach to Pursue PCI DSS Co...</td>\n",
+       "      <td>Las nuevas recomendaciones para las evaluacion...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>35</th>\n",
+       "      <td>orca-2-13b</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>more on penetration testing</td>\n",
+       "      <td>¿Puedes dar más información sobre las pruebas ...</td>\n",
+       "      <td>[The Prioritized Approach to Pursue PCI DSS Co...</td>\n",
+       "      <td>Sí, puedo dar más información sobre las prueba...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>36</th>\n",
+       "      <td>llama-2-13b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>What's PCI DSS?</td>\n",
+       "      <td></td>\n",
+       "      <td>[PCI Data Security Standard\\nPCI DSS provides ...</td>\n",
+       "      <td>PCI DSS is the Payment Card Industry Data Secu...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>37</th>\n",
+       "      <td>llama-2-13b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>Can you summarize the changes made from PCI DS...</td>\n",
+       "      <td>What are the key changes between PCI DSS versi...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>What are the key changes between PCI DSS vers...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>38</th>\n",
+       "      <td>llama-2-13b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>new requirements for vulnerability assessments</td>\n",
+       "      <td>What are the new requirements for vulnerabilit...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>According to the provided document, there are ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>39</th>\n",
+       "      <td>llama-2-13b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>more on penetration testing</td>\n",
+       "      <td>What are the new requirements for penetration ...</td>\n",
+       "      <td>[PCI DSS v4.0 ROC Template r1  December 2022  ...</td>\n",
+       "      <td>Based on the provided information, the new re...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>40</th>\n",
+       "      <td>llama-2-13b</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>What's PCI DSS?</td>\n",
+       "      <td></td>\n",
+       "      <td>[PCI Data Security Standard\\nPCI DSS provides ...</td>\n",
+       "      <td>PCI DSS stands for Payment Card Industry Data ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>41</th>\n",
+       "      <td>llama-2-13b</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>Can you summarize the changes made from PCI DS...</td>\n",
+       "      <td>What are the key changes between PCI DSS versi...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>Based on the provided document, here are the ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>42</th>\n",
+       "      <td>llama-2-13b</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>new requirements for vulnerability assessments</td>\n",
+       "      <td>What are the new requirements for vulnerabilit...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>According to the provided document, there are...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>43</th>\n",
+       "      <td>llama-2-13b</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>more on penetration testing</td>\n",
+       "      <td>What are the new requirements for penetration ...</td>\n",
+       "      <td>[PCI DSS v4.0 ROC Template r1  December 2022  ...</td>\n",
+       "      <td>Based on the provided information, there are ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>44</th>\n",
+       "      <td>llama-2-13b</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>What's PCI DSS?</td>\n",
+       "      <td></td>\n",
+       "      <td>[PCI Data Security Standard\\nPCI DSS provides ...</td>\n",
+       "      <td>PCI DSS stands for Payment Card Industry Data ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>45</th>\n",
+       "      <td>llama-2-13b</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>Can you summarize the changes made from PCI DS...</td>\n",
+       "      <td>What are the key changes between PCI DSS versi...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>Based on the provided document, here are some...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>46</th>\n",
+       "      <td>llama-2-13b</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>new requirements for vulnerability assessments</td>\n",
+       "      <td>What are the new requirements for vulnerabilit...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>According to the provided document, there are...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>47</th>\n",
+       "      <td>llama-2-13b</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>more on penetration testing</td>\n",
+       "      <td>What are the new requirements for penetration ...</td>\n",
+       "      <td>[PCI DSS v4.0 ROC Template r1  December 2022  ...</td>\n",
+       "      <td>Based on the provided information, there are ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>48</th>\n",
+       "      <td>gpt-3.5-turbo</td>\n",
+       "      <td></td>\n",
+       "      <td>What's PCI DSS?</td>\n",
+       "      <td></td>\n",
+       "      <td>[PCI Data Security Standard\\nPCI DSS provides ...</td>\n",
+       "      <td>PCI DSS stands for Payment Card Industry Data ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>49</th>\n",
+       "      <td>gpt-3.5-turbo</td>\n",
+       "      <td></td>\n",
+       "      <td>Can you summarize the changes made from PCI DS...</td>\n",
+       "      <td>What are the differences between PCI DSS versi...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>The document provided does not specify the spe...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>50</th>\n",
+       "      <td>gpt-3.5-turbo</td>\n",
+       "      <td></td>\n",
+       "      <td>new requirements for vulnerability assessments</td>\n",
+       "      <td>What are the new requirements for vulnerabilit...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>In PCI DSS version 4.0, there are several new ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>51</th>\n",
+       "      <td>gpt-3.5-turbo</td>\n",
+       "      <td></td>\n",
+       "      <td>more on penetration testing</td>\n",
+       "      <td>Can you provide more information about the cha...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>I'm sorry, but I don't have access to the spec...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>52</th>\n",
+       "      <td>gpt-3.5-turbo-instruct</td>\n",
+       "      <td></td>\n",
+       "      <td>What's PCI DSS?</td>\n",
+       "      <td></td>\n",
+       "      <td>[PCI Data Security Standard\\nPCI DSS provides ...</td>\n",
+       "      <td>The PCI Data Security Standard (PCI DSS) is a...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>53</th>\n",
+       "      <td>gpt-3.5-turbo-instruct</td>\n",
+       "      <td></td>\n",
+       "      <td>Can you summarize the changes made from PCI DS...</td>\n",
+       "      <td>What changes were made from PCI DSS version 3...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>The changes made from PCI DSS version 3.2.1 t...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>54</th>\n",
+       "      <td>gpt-3.5-turbo-instruct</td>\n",
+       "      <td></td>\n",
+       "      <td>new requirements for vulnerability assessments</td>\n",
+       "      <td>What are the new requirements for vulnerabili...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>The new requirements for vulnerability assess...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>55</th>\n",
+       "      <td>gpt-3.5-turbo-instruct</td>\n",
+       "      <td></td>\n",
+       "      <td>more on penetration testing</td>\n",
+       "      <td>Can you provide more information about the ne...</td>\n",
+       "      <td>[PCI DSS v4.0 ROC Template r1  December 2022  ...</td>\n",
+       "      <td>The new requirements for penetration testing ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>56</th>\n",
+       "      <td>gpt-4</td>\n",
+       "      <td></td>\n",
+       "      <td>What's PCI DSS?</td>\n",
+       "      <td></td>\n",
+       "      <td>[PCI Data Security Standard\\nPCI DSS provides ...</td>\n",
+       "      <td>The PCI Data Security Standard (PCI DSS) is a ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>57</th>\n",
+       "      <td>gpt-4</td>\n",
+       "      <td></td>\n",
+       "      <td>Can you summarize the changes made from PCI DS...</td>\n",
+       "      <td>Can you provide a summary of the changes that ...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>I'm sorry, but the provided context does not i...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>58</th>\n",
+       "      <td>gpt-4</td>\n",
+       "      <td></td>\n",
+       "      <td>new requirements for vulnerability assessments</td>\n",
+       "      <td>What are the new requirements for vulnerabilit...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>The new requirements for vulnerability assessm...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>59</th>\n",
+       "      <td>gpt-4</td>\n",
+       "      <td></td>\n",
+       "      <td>more on penetration testing</td>\n",
+       "      <td>Can you provide more information on penetratio...</td>\n",
+       "      <td>[PCI DSS v4.0 ROC Template r1  December 2022  ...</td>\n",
+       "      <td>Penetration testing in the context of PCI DSS ...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                model_name repetition_penalty  \\\n",
+       "0                orca-2-7b               1.05   \n",
+       "1                orca-2-7b               1.05   \n",
+       "2                orca-2-7b               1.05   \n",
+       "3                orca-2-7b               1.05   \n",
+       "4                orca-2-7b               1.10   \n",
+       "5                orca-2-7b               1.10   \n",
+       "6                orca-2-7b               1.10   \n",
+       "7                orca-2-7b               1.10   \n",
+       "8                orca-2-7b               1.15   \n",
+       "9                orca-2-7b               1.15   \n",
+       "10               orca-2-7b               1.15   \n",
+       "11               orca-2-7b               1.15   \n",
+       "12              llama-2-7b               1.05   \n",
+       "13              llama-2-7b               1.05   \n",
+       "14              llama-2-7b               1.05   \n",
+       "15              llama-2-7b               1.05   \n",
+       "16              llama-2-7b               1.10   \n",
+       "17              llama-2-7b               1.10   \n",
+       "18              llama-2-7b               1.10   \n",
+       "19              llama-2-7b               1.10   \n",
+       "20              llama-2-7b               1.15   \n",
+       "21              llama-2-7b               1.15   \n",
+       "22              llama-2-7b               1.15   \n",
+       "23              llama-2-7b               1.15   \n",
+       "24              orca-2-13b               1.05   \n",
+       "25              orca-2-13b               1.05   \n",
+       "26              orca-2-13b               1.05   \n",
+       "27              orca-2-13b               1.05   \n",
+       "28              orca-2-13b               1.10   \n",
+       "29              orca-2-13b               1.10   \n",
+       "30              orca-2-13b               1.10   \n",
+       "31              orca-2-13b               1.10   \n",
+       "32              orca-2-13b               1.15   \n",
+       "33              orca-2-13b               1.15   \n",
+       "34              orca-2-13b               1.15   \n",
+       "35              orca-2-13b               1.15   \n",
+       "36             llama-2-13b               1.05   \n",
+       "37             llama-2-13b               1.05   \n",
+       "38             llama-2-13b               1.05   \n",
+       "39             llama-2-13b               1.05   \n",
+       "40             llama-2-13b               1.10   \n",
+       "41             llama-2-13b               1.10   \n",
+       "42             llama-2-13b               1.10   \n",
+       "43             llama-2-13b               1.10   \n",
+       "44             llama-2-13b               1.15   \n",
+       "45             llama-2-13b               1.15   \n",
+       "46             llama-2-13b               1.15   \n",
+       "47             llama-2-13b               1.15   \n",
+       "48           gpt-3.5-turbo                      \n",
+       "49           gpt-3.5-turbo                      \n",
+       "50           gpt-3.5-turbo                      \n",
+       "51           gpt-3.5-turbo                      \n",
+       "52  gpt-3.5-turbo-instruct                      \n",
+       "53  gpt-3.5-turbo-instruct                      \n",
+       "54  gpt-3.5-turbo-instruct                      \n",
+       "55  gpt-3.5-turbo-instruct                      \n",
+       "56                   gpt-4                      \n",
+       "57                   gpt-4                      \n",
+       "58                   gpt-4                      \n",
+       "59                   gpt-4                      \n",
+       "\n",
+       "                                        user_question  \\\n",
+       "0                                     What's PCI DSS?   \n",
+       "1   Can you summarize the changes made from PCI DS...   \n",
+       "2      new requirements for vulnerability assessments   \n",
+       "3                         more on penetration testing   \n",
+       "4                                     What's PCI DSS?   \n",
+       "5   Can you summarize the changes made from PCI DS...   \n",
+       "6      new requirements for vulnerability assessments   \n",
+       "7                         more on penetration testing   \n",
+       "8                                     What's PCI DSS?   \n",
+       "9   Can you summarize the changes made from PCI DS...   \n",
+       "10     new requirements for vulnerability assessments   \n",
+       "11                        more on penetration testing   \n",
+       "12                                    What's PCI DSS?   \n",
+       "13  Can you summarize the changes made from PCI DS...   \n",
+       "14     new requirements for vulnerability assessments   \n",
+       "15                        more on penetration testing   \n",
+       "16                                    What's PCI DSS?   \n",
+       "17  Can you summarize the changes made from PCI DS...   \n",
+       "18     new requirements for vulnerability assessments   \n",
+       "19                        more on penetration testing   \n",
+       "20                                    What's PCI DSS?   \n",
+       "21  Can you summarize the changes made from PCI DS...   \n",
+       "22     new requirements for vulnerability assessments   \n",
+       "23                        more on penetration testing   \n",
+       "24                                    What's PCI DSS?   \n",
+       "25  Can you summarize the changes made from PCI DS...   \n",
+       "26     new requirements for vulnerability assessments   \n",
+       "27                        more on penetration testing   \n",
+       "28                                    What's PCI DSS?   \n",
+       "29  Can you summarize the changes made from PCI DS...   \n",
+       "30     new requirements for vulnerability assessments   \n",
+       "31                        more on penetration testing   \n",
+       "32                                    What's PCI DSS?   \n",
+       "33  Can you summarize the changes made from PCI DS...   \n",
+       "34     new requirements for vulnerability assessments   \n",
+       "35                        more on penetration testing   \n",
+       "36                                    What's PCI DSS?   \n",
+       "37  Can you summarize the changes made from PCI DS...   \n",
+       "38     new requirements for vulnerability assessments   \n",
+       "39                        more on penetration testing   \n",
+       "40                                    What's PCI DSS?   \n",
+       "41  Can you summarize the changes made from PCI DS...   \n",
+       "42     new requirements for vulnerability assessments   \n",
+       "43                        more on penetration testing   \n",
+       "44                                    What's PCI DSS?   \n",
+       "45  Can you summarize the changes made from PCI DS...   \n",
+       "46     new requirements for vulnerability assessments   \n",
+       "47                        more on penetration testing   \n",
+       "48                                    What's PCI DSS?   \n",
+       "49  Can you summarize the changes made from PCI DS...   \n",
+       "50     new requirements for vulnerability assessments   \n",
+       "51                        more on penetration testing   \n",
+       "52                                    What's PCI DSS?   \n",
+       "53  Can you summarize the changes made from PCI DS...   \n",
+       "54     new requirements for vulnerability assessments   \n",
+       "55                        more on penetration testing   \n",
+       "56                                    What's PCI DSS?   \n",
+       "57  Can you summarize the changes made from PCI DS...   \n",
+       "58     new requirements for vulnerability assessments   \n",
+       "59                        more on penetration testing   \n",
+       "\n",
+       "                                  standalone_question  \\\n",
+       "0                                                       \n",
+       "1   What are the main differences between PCI DSS ...   \n",
+       "2   What are the new requirements for vulnerabilit...   \n",
+       "3   What are the new requirements for penetration ...   \n",
+       "4                                                       \n",
+       "5   What are the main differences between PCI DSS ...   \n",
+       "6   What are the new requirements for vulnerabilit...   \n",
+       "7   What are some key differences between penetrat...   \n",
+       "8                                                       \n",
+       "9   How has the latest version of PCI DSS, version...   \n",
+       "10  What are some new requirements for vulnerabili...   \n",
+       "11  What are some best practices for conducting ef...   \n",
+       "12                                                      \n",
+       "13  What are the key changes made in PCI DSS versi...   \n",
+       "14  What are the new requirements for vulnerabilit...   \n",
+       "15  What are the new requirements for penetration ...   \n",
+       "16                                                      \n",
+       "17  What are the key changes made in PCI DSS versi...   \n",
+       "18  What are the new requirements for vulnerabilit...   \n",
+       "19  What are the new requirements for penetration ...   \n",
+       "20                                                      \n",
+       "21  What are the key changes between PCI DSS versi...   \n",
+       "22  What are some of the new requirements for vuln...   \n",
+       "23  Could you explain what penetration testing ent...   \n",
+       "24                                                      \n",
+       "25  ¿Puedes resumir los cambios realizados desde l...   \n",
+       "26  ¿Cuáles son las nuevas requisitos para las eva...   \n",
+       "27  ¿Puedes dar más detalles sobre las prácticas d...   \n",
+       "28                                                      \n",
+       "29  ¿Puedes resumir los cambios realizados desde l...   \n",
+       "30  ¿Cuáles son las nuevas exigencias para las eva...   \n",
+       "31  ¿Puedo obtener más información sobre las prueb...   \n",
+       "32                                                      \n",
+       "33  ¿Puedes resumir los cambios realizados desde l...   \n",
+       "34  ¿Cuáles son las nuevas recomendaciones para la...   \n",
+       "35  ¿Puedes dar más información sobre las pruebas ...   \n",
+       "36                                                      \n",
+       "37  What are the key changes between PCI DSS versi...   \n",
+       "38  What are the new requirements for vulnerabilit...   \n",
+       "39  What are the new requirements for penetration ...   \n",
+       "40                                                      \n",
+       "41  What are the key changes between PCI DSS versi...   \n",
+       "42  What are the new requirements for vulnerabilit...   \n",
+       "43  What are the new requirements for penetration ...   \n",
+       "44                                                      \n",
+       "45  What are the key changes between PCI DSS versi...   \n",
+       "46  What are the new requirements for vulnerabilit...   \n",
+       "47  What are the new requirements for penetration ...   \n",
+       "48                                                      \n",
+       "49  What are the differences between PCI DSS versi...   \n",
+       "50  What are the new requirements for vulnerabilit...   \n",
+       "51  Can you provide more information about the cha...   \n",
+       "52                                                      \n",
+       "53   What changes were made from PCI DSS version 3...   \n",
+       "54   What are the new requirements for vulnerabili...   \n",
+       "55   Can you provide more information about the ne...   \n",
+       "56                                                      \n",
+       "57  Can you provide a summary of the changes that ...   \n",
+       "58  What are the new requirements for vulnerabilit...   \n",
+       "59  Can you provide more information on penetratio...   \n",
+       "\n",
+       "                                             contexts  \\\n",
+       "0   [PCI Data Security Standard\\nPCI DSS provides ...   \n",
+       "1   [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "2   [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "3   [PCI DSS v4.0 ROC Template r1  December 2022  ...   \n",
+       "4   [PCI Data Security Standard\\nPCI DSS provides ...   \n",
+       "5   [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "6   [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "7   [PCI DSS v4.0 ROC Template r1  December 2022  ...   \n",
+       "8   [PCI Data Security Standard\\nPCI DSS provides ...   \n",
+       "9   [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "10  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "11  [methodology  examined for this testing \\nproc...   \n",
+       "12  [PCI Data Security Standard\\nPCI DSS provides ...   \n",
+       "13  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "14  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "15  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "16  [PCI Data Security Standard\\nPCI DSS provides ...   \n",
+       "17  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "18  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "19  [PCI DSS v4.0 ROC Template r1  December 2022  ...   \n",
+       "20  [PCI Data Security Standard\\nPCI DSS provides ...   \n",
+       "21  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "22  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "23  [PCI DSS v4.0 ROC Template r1  December 2022  ...   \n",
+       "24  [PCI Data Security Standard\\nPCI DSS provides ...   \n",
+       "25  [April 2015  PCI DSS 3.1, \\nRevision  1.0 Revi...   \n",
+       "26  [The Prioritized Approach to Pursue PCI DSS Co...   \n",
+       "27  [The Prioritized Approach to Pursue PCI DSS Co...   \n",
+       "28  [PCI Data Security Standard\\nPCI DSS provides ...   \n",
+       "29  [April 2015  PCI DSS 3.1, \\nRevision  1.0 Revi...   \n",
+       "30  [The Prioritized Approach to Pursue PCI DSS Co...   \n",
+       "31  [The Prioritized Approach to Pursue PCI DSS Co...   \n",
+       "32  [PCI Data Security Standard\\nPCI DSS provides ...   \n",
+       "33  [April 2015  PCI DSS 3.1, \\nRevision  1.0 Revi...   \n",
+       "34  [The Prioritized Approach to Pursue PCI DSS Co...   \n",
+       "35  [The Prioritized Approach to Pursue PCI DSS Co...   \n",
+       "36  [PCI Data Security Standard\\nPCI DSS provides ...   \n",
+       "37  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "38  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "39  [PCI DSS v4.0 ROC Template r1  December 2022  ...   \n",
+       "40  [PCI Data Security Standard\\nPCI DSS provides ...   \n",
+       "41  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "42  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "43  [PCI DSS v4.0 ROC Template r1  December 2022  ...   \n",
+       "44  [PCI Data Security Standard\\nPCI DSS provides ...   \n",
+       "45  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "46  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "47  [PCI DSS v4.0 ROC Template r1  December 2022  ...   \n",
+       "48  [PCI Data Security Standard\\nPCI DSS provides ...   \n",
+       "49  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "50  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "51  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "52  [PCI Data Security Standard\\nPCI DSS provides ...   \n",
+       "53  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "54  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "55  [PCI DSS v4.0 ROC Template r1  December 2022  ...   \n",
+       "56  [PCI Data Security Standard\\nPCI DSS provides ...   \n",
+       "57  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "58  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "59  [PCI DSS v4.0 ROC Template r1  December 2022  ...   \n",
+       "\n",
+       "                                               answer  \n",
+       "0   PCI DSS stands for Payment Card Industry Data ...  \n",
+       "1   The main differences between PCI DSS version 3...  \n",
+       "2   The new requirements for vulnerability assessm...  \n",
+       "3   The new requirements for penetration testing i...  \n",
+       "4   PCI DSS stands for Payment Card Industry Data ...  \n",
+       "5   The main differences between PCI DSS versions ...  \n",
+       "6   There are several new requirements related to ...  \n",
+       "7   Some key differences between penetration testi...  \n",
+       "8   PCI DSS stands for Payment Card Industry Data ...  \n",
+       "9   The latest version of PCI DSS, version 4.0, di...  \n",
+       "10  Some new requirements for vulnerability assess...  \n",
+       "11  Some best practices for conducting effective p...  \n",
+       "12  PCI DSS stands for Payment Card Industry Data ...  \n",
+       "13   Based on the provided document, the key chang...  \n",
+       "14   According to the provided summary of changes,...  \n",
+       "15   The new requirement for penetration testing i...  \n",
+       "16  PCI DSS stands for Payment Card Industry Data ...  \n",
+       "17  The key changes made in PCI DSS version 4.0 co...  \n",
+       "18  The new requirement for vulnerability assessme...  \n",
+       "19  The new requirements for penetration testing i...  \n",
+       "20  According to the given quick reference guide, ...  \n",
+       "21  The document highlights several significant mo...  \n",
+       "22  According to the provided documents, one of th...  \n",
+       "23  Sure! Penetration testing is a crucial compone...  \n",
+       "24  PCI DSS is a global standard that provides a b...  \n",
+       "25  Sí, puedo resumir los cambios realizados desde...  \n",
+       "26  Las nuevas requisitos para las evaluaciones de...  \n",
+       "27  Sí, puedo dar más detalles sobre las prácticas...  \n",
+       "28  PCI DSS is a set of requirements designed to p...  \n",
+       "29  Sí, puedo resumir los cambios realizados desde...  \n",
+       "30  Las nuevas exigencias para las evaluaciones de...  \n",
+       "31  Yes, you can get more information about penetr...  \n",
+       "32  PCI DSS is a set of requirements designed to p...  \n",
+       "33  Sí, puedo resumir los cambios realizados desde...  \n",
+       "34  Las nuevas recomendaciones para las evaluacion...  \n",
+       "35  Sí, puedo dar más información sobre las prueba...  \n",
+       "36  PCI DSS is the Payment Card Industry Data Secu...  \n",
+       "37   What are the key changes between PCI DSS vers...  \n",
+       "38  According to the provided document, there are ...  \n",
+       "39   Based on the provided information, the new re...  \n",
+       "40  PCI DSS stands for Payment Card Industry Data ...  \n",
+       "41   Based on the provided document, here are the ...  \n",
+       "42   According to the provided document, there are...  \n",
+       "43   Based on the provided information, there are ...  \n",
+       "44  PCI DSS stands for Payment Card Industry Data ...  \n",
+       "45   Based on the provided document, here are some...  \n",
+       "46   According to the provided document, there are...  \n",
+       "47   Based on the provided information, there are ...  \n",
+       "48  PCI DSS stands for Payment Card Industry Data ...  \n",
+       "49  The document provided does not specify the spe...  \n",
+       "50  In PCI DSS version 4.0, there are several new ...  \n",
+       "51  I'm sorry, but I don't have access to the spec...  \n",
+       "52   The PCI Data Security Standard (PCI DSS) is a...  \n",
+       "53   The changes made from PCI DSS version 3.2.1 t...  \n",
+       "54   The new requirements for vulnerability assess...  \n",
+       "55   The new requirements for penetration testing ...  \n",
+       "56  The PCI Data Security Standard (PCI DSS) is a ...  \n",
+       "57  I'm sorry, but the provided context does not i...  \n",
+       "58  The new requirements for vulnerability assessm...  \n",
+       "59  Penetration testing in the context of PCI DSS ...  "
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "raw_pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "raw_pd.to_excel(\"./results/raw_data.xlsx\", index=False)\n",
+    "perf_pd.to_excel(\"./results/perf_data.xlsx\", index=False)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

results/2_process-raw-data_nvidia-4080.ipynb

@@ -0,0 +1,1355 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "workding dir: /home/inflaton/code/projects/chat-with-orca-2\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import sys\n",
+    "from pathlib import Path\n",
+    "\n",
+    "workding_dir = str(Path.cwd().parent)\n",
+    "os.chdir(workding_dir)\n",
+    "sys.path.append(workding_dir)\n",
+    "print(\"workding dir:\", workding_dir)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading env vars from: /home/inflaton/code/projects/chat-with-orca-2/.env\n",
+      "Running on:  Linux-5.10.102.1-microsoft-standard-WSL2-x86_64-with-glibc2.35\n",
+      "MPS is NOT available\n",
+      "CUDA is  available\n",
+      "MPS not available because the current PyTorch install was not built with MPS enabled.\n",
+      "CUDA is available, we have found  1  GPU(s)\n",
+      "NVIDIA GeForce RTX 4080 Laptop GPU\n",
+      "CUDA version: 12.1\n",
+      "hf_embeddings_device_type: cuda:0\n",
+      "hf_pipeline_device_type: cuda:0\n",
+      "load INSTRUCTOR_Transformer\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/inflaton/miniconda3/envs/chat-with-orca-2/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "max_seq_length  512\n",
+      "Completed in 0.638s\n",
+      "Load index from ./data/faiss_1024_512/ with FAISS\n",
+      "Completed in 0.020s\n",
+      "initializing LLM: openai\n",
+      "  hf_pipeline_device_type: cuda:0\n",
+      "     load_quantized_model: None\n",
+      "              torch_dtype: torch.float16\n",
+      "                 n_threds: 4\n",
+      "              using model: gpt-3.5-turbo\n",
+      "initialization complete\n",
+      "Completed in 0.028s\n",
+      "CPU times: user 6.16 s, sys: 4.73 s, total: 10.9 s\n",
+      "Wall time: 3.62 s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "os.environ[\"LLM_MODEL_TYPE\"] = \"openai\"\n",
+    "from eval_models import *"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "loading:  ./notebook/nvidia-4080-llama-2-7b.ipynb\n",
+      "processing repetition_penalty: 1.05\n",
+      "processing repetition_penalty: 1.10\n",
+      "processing repetition_penalty: 1.15\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating: 100%|██████████| 8/8 [00:49<00:00,  6.14s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "# Ragas scores: {'faithfulness': 1.0000, 'answer_relevancy': 0.7320, 'overall_score': 0.8452}\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating: 100%|██████████| 8/8 [00:40<00:00,  5.12s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "# Ragas scores: {'faithfulness': 0.7500, 'answer_relevancy': 0.7224, 'overall_score': 0.7359}\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating: 100%|██████████| 8/8 [00:38<00:00,  4.80s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "# Ragas scores: {'faithfulness': 0.7556, 'answer_relevancy': 0.9682, 'overall_score': 0.8487}\n",
+      "\n",
+      "loading:  ./notebook/nvidia-4080-orca-2-7b.ipynb\n",
+      "processing repetition_penalty: 1.05\n",
+      "processing repetition_penalty: 1.10\n",
+      "processing repetition_penalty: 1.15\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating: 100%|███���██████| 8/8 [00:42<00:00,  5.25s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "# Ragas scores: {'faithfulness': 0.8304, 'answer_relevancy': 0.9783, 'overall_score': 0.8983}\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating: 100%|██████████| 8/8 [00:59<00:00,  7.44s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "# Ragas scores: {'faithfulness': 1.0000, 'answer_relevancy': 0.9711, 'overall_score': 0.9853}\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating: 100%|██████████| 8/8 [00:33<00:00,  4.18s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "# Ragas scores: {'faithfulness': 0.8667, 'answer_relevancy': 0.9637, 'overall_score': 0.9126}\n",
+      "\n",
+      "loading:  ./notebook/nvidia-4080-orca-2-7b_tgi.ipynb\n",
+      "processing repetition_penalty: 1.05\n",
+      "processing repetition_penalty: 1.10\n",
+      "processing repetition_penalty: 1.15\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating: 100%|██████████| 8/8 [00:54<00:00,  6.86s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "# Ragas scores: {'faithfulness': 1.0000, 'answer_relevancy': 0.9676, 'overall_score': 0.9835}\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating: 100%|██████████| 8/8 [01:02<00:00,  7.87s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "# Ragas scores: {'faithfulness': 0.8583, 'answer_relevancy': 0.9557, 'overall_score': 0.9044}\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating: 100%|██████████| 8/8 [00:48<00:00,  6.08s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "# Ragas scores: {'faithfulness': 0.7500, 'answer_relevancy': 0.7140, 'overall_score': 0.7316}\n",
+      "\n",
+      "loading:  ./notebook/nvidia-4080-orca-2-13b_tgi.ipynb\n",
+      "processing repetition_penalty: 1.05\n",
+      "processing repetition_penalty: 1.10\n",
+      "processing repetition_penalty: 1.15\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating: 100%|██████████| 8/8 [00:48<00:00,  6.01s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "# Ragas scores: {'faithfulness': 0.9500, 'answer_relevancy': 0.9637, 'overall_score': 0.9568}\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating: 100%|██████████| 8/8 [00:35<00:00,  4.48s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "# Ragas scores: {'faithfulness': 0.7500, 'answer_relevancy': 0.9468, 'overall_score': 0.8370}\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating: 100%|██████████| 8/8 [00:41<00:00,  5.24s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "# Ragas scores: {'faithfulness': 1.0000, 'answer_relevancy': 0.9388, 'overall_score': 0.9684}\n",
+      "\n",
+      "CPU times: user 6.76 s, sys: 448 ms, total: 7.21 s\n",
+      "Wall time: 9min 26s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "model_names = [\"llama-2-7b\", \"orca-2-7b\", \"orca-2-7b_tgi\", \"orca-2-13b_tgi\"]\n",
+    "\n",
+    "perf_pd, raw_pd = evaluate_models(model_names, prefix=\"nvidia-4080\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>model_name</th>\n",
+       "      <th>repetition_penalty</th>\n",
+       "      <th>faithfulness</th>\n",
+       "      <th>answer_relevancy</th>\n",
+       "      <th>overall_score</th>\n",
+       "      <th>total_time_used</th>\n",
+       "      <th>num_tokens_generated</th>\n",
+       "      <th>token_per_second</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>llama-2-7b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>0.731964</td>\n",
+       "      <td>0.845241</td>\n",
+       "      <td>1234.418</td>\n",
+       "      <td>2457</td>\n",
+       "      <td>1.990</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>llama-2-7b</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>0.750000</td>\n",
+       "      <td>0.722392</td>\n",
+       "      <td>0.735937</td>\n",
+       "      <td>272.474</td>\n",
+       "      <td>623</td>\n",
+       "      <td>2.286</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>llama-2-7b</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>0.755556</td>\n",
+       "      <td>0.968164</td>\n",
+       "      <td>0.848748</td>\n",
+       "      <td>319.531</td>\n",
+       "      <td>792</td>\n",
+       "      <td>2.479</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>orca-2-7b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>0.830357</td>\n",
+       "      <td>0.978326</td>\n",
+       "      <td>0.898289</td>\n",
+       "      <td>254.981</td>\n",
+       "      <td>536</td>\n",
+       "      <td>2.102</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>orca-2-7b</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>0.971117</td>\n",
+       "      <td>0.985347</td>\n",
+       "      <td>273.518</td>\n",
+       "      <td>549</td>\n",
+       "      <td>2.007</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>orca-2-7b</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>0.866667</td>\n",
+       "      <td>0.963659</td>\n",
+       "      <td>0.912593</td>\n",
+       "      <td>214.116</td>\n",
+       "      <td>432</td>\n",
+       "      <td>2.018</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>orca-2-7b_tgi</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>0.967557</td>\n",
+       "      <td>0.983511</td>\n",
+       "      <td>330.008</td>\n",
+       "      <td>587</td>\n",
+       "      <td>1.779</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>orca-2-7b_tgi</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>0.858333</td>\n",
+       "      <td>0.955750</td>\n",
+       "      <td>0.904426</td>\n",
+       "      <td>304.184</td>\n",
+       "      <td>502</td>\n",
+       "      <td>1.650</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>orca-2-7b_tgi</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>0.750000</td>\n",
+       "      <td>0.714001</td>\n",
+       "      <td>0.731558</td>\n",
+       "      <td>203.892</td>\n",
+       "      <td>339</td>\n",
+       "      <td>1.663</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>orca-2-13b_tgi</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>0.950000</td>\n",
+       "      <td>0.963749</td>\n",
+       "      <td>0.956825</td>\n",
+       "      <td>30.847</td>\n",
+       "      <td>455</td>\n",
+       "      <td>14.750</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>10</th>\n",
+       "      <td>orca-2-13b_tgi</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>0.750000</td>\n",
+       "      <td>0.946751</td>\n",
+       "      <td>0.836968</td>\n",
+       "      <td>24.490</td>\n",
+       "      <td>348</td>\n",
+       "      <td>14.210</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>11</th>\n",
+       "      <td>orca-2-13b_tgi</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>0.938756</td>\n",
+       "      <td>0.968411</td>\n",
+       "      <td>42.575</td>\n",
+       "      <td>649</td>\n",
+       "      <td>15.244</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "        model_name repetition_penalty  faithfulness  answer_relevancy  \\\n",
+       "0       llama-2-7b               1.05      1.000000          0.731964   \n",
+       "1       llama-2-7b               1.10      0.750000          0.722392   \n",
+       "2       llama-2-7b               1.15      0.755556          0.968164   \n",
+       "3        orca-2-7b               1.05      0.830357          0.978326   \n",
+       "4        orca-2-7b               1.10      1.000000          0.971117   \n",
+       "5        orca-2-7b               1.15      0.866667          0.963659   \n",
+       "6    orca-2-7b_tgi               1.05      1.000000          0.967557   \n",
+       "7    orca-2-7b_tgi               1.10      0.858333          0.955750   \n",
+       "8    orca-2-7b_tgi               1.15      0.750000          0.714001   \n",
+       "9   orca-2-13b_tgi               1.05      0.950000          0.963749   \n",
+       "10  orca-2-13b_tgi               1.10      0.750000          0.946751   \n",
+       "11  orca-2-13b_tgi               1.15      1.000000          0.938756   \n",
+       "\n",
+       "    overall_score  total_time_used  num_tokens_generated  token_per_second  \n",
+       "0        0.845241         1234.418                  2457             1.990  \n",
+       "1        0.735937          272.474                   623             2.286  \n",
+       "2        0.848748          319.531                   792             2.479  \n",
+       "3        0.898289          254.981                   536             2.102  \n",
+       "4        0.985347          273.518                   549             2.007  \n",
+       "5        0.912593          214.116                   432             2.018  \n",
+       "6        0.983511          330.008                   587             1.779  \n",
+       "7        0.904426          304.184                   502             1.650  \n",
+       "8        0.731558          203.892                   339             1.663  \n",
+       "9        0.956825           30.847                   455            14.750  \n",
+       "10       0.836968           24.490                   348            14.210  \n",
+       "11       0.968411           42.575                   649            15.244  "
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "perf_pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>model_name</th>\n",
+       "      <th>repetition_penalty</th>\n",
+       "      <th>user_question</th>\n",
+       "      <th>standalone_question</th>\n",
+       "      <th>contexts</th>\n",
+       "      <th>answer</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>llama-2-7b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>What's PCI DSS?</td>\n",
+       "      <td></td>\n",
+       "      <td>[PCI Data Security Standard\\nPCI DSS provides ...</td>\n",
+       "      <td>PCI DSS stands for Payment Card Industry Data ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>llama-2-7b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>Can you summarize the changes made from PCI DS...</td>\n",
+       "      <td>What are the key changes made in PCI DSS versi...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>The key changes made in PCI DSS version 4.0 c...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>llama-2-7b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>new requirements for vulnerability assessments</td>\n",
+       "      <td>What are the updated requirements for vulnerab...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>The updated requirements for vulnerability as...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>llama-2-7b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>more on penetration testing</td>\n",
+       "      <td>What are the updated requirements for vulnerab...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>The updated requirements for vulnerability as...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>llama-2-7b</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>What's PCI DSS?</td>\n",
+       "      <td></td>\n",
+       "      <td>[PCI Data Security Standard\\nPCI DSS provides ...</td>\n",
+       "      <td>PCI DSS stands for Payment Card Industry Data ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>llama-2-7b</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>Can you summarize the changes made from PCI DS...</td>\n",
+       "      <td>What are the key changes made in PCI DSS versi...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>The key changes made in PCI DSS version 4.0 c...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>llama-2-7b</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>new requirements for vulnerability assessments</td>\n",
+       "      <td>What are the new requirements for vulnerabilit...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>According to the provided summary of changes,...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>llama-2-7b</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>more on penetration testing</td>\n",
+       "      <td>What are the key differences between penetrati...</td>\n",
+       "      <td>[PCI DSS v4.0 ROC Template r1  December 2022  ...</td>\n",
+       "      <td>Penetration testing and vulnerability assessme...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>llama-2-7b</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>What's PCI DSS?</td>\n",
+       "      <td></td>\n",
+       "      <td>[PCI Data Security Standard\\nPCI DSS provides ...</td>\n",
+       "      <td>PCI DSS stands for Payment Card Industry Data ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>llama-2-7b</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>Can you summarize the changes made from PCI DS...</td>\n",
+       "      <td>What are the key changes between PCI DSS versi...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>The key changes between PCI DSS versions 3.2.1...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>10</th>\n",
+       "      <td>llama-2-7b</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>new requirements for vulnerability assessments</td>\n",
+       "      <td>What are some of the new requirements for vuln...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>In PCI DSS version 4.0, there are several new ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>11</th>\n",
+       "      <td>llama-2-7b</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>more on penetration testing</td>\n",
+       "      <td>What are some common techniques used in penetr...</td>\n",
+       "      <td>[circumvent or defeat the security features of...</td>\n",
+       "      <td>Penetration testing involves various technique...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>12</th>\n",
+       "      <td>orca-2-7b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>What's PCI DSS?</td>\n",
+       "      <td></td>\n",
+       "      <td>[PCI Data Security Standard\\nPCI DSS provides ...</td>\n",
+       "      <td>PCI DSS stands for Payment Card Industry Data ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13</th>\n",
+       "      <td>orca-2-7b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>Can you summarize the changes made from PCI DS...</td>\n",
+       "      <td>What are the main differences between PCI DSS ...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>The main differences between PCI DSS version 3...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>14</th>\n",
+       "      <td>orca-2-7b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>new requirements for vulnerability assessments</td>\n",
+       "      <td>What are the new requirements for vulnerabilit...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>The new requirements for vulnerability assessm...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>15</th>\n",
+       "      <td>orca-2-7b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>more on penetration testing</td>\n",
+       "      <td>What are the new requirements for penetration ...</td>\n",
+       "      <td>[PCI DSS v4.0 ROC Template r1  December 2022  ...</td>\n",
+       "      <td>The new requirements for penetration testing i...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>16</th>\n",
+       "      <td>orca-2-7b</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>What's PCI DSS?</td>\n",
+       "      <td></td>\n",
+       "      <td>[PCI Data Security Standard\\nPCI DSS provides ...</td>\n",
+       "      <td>PCI DSS stands for Payment Card Industry Data ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>17</th>\n",
+       "      <td>orca-2-7b</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>Can you summarize the changes made from PCI DS...</td>\n",
+       "      <td>What are the main differences between PCI DSS ...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>The main differences between PCI DSS versions ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>18</th>\n",
+       "      <td>orca-2-7b</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>new requirements for vulnerability assessments</td>\n",
+       "      <td>What are the new requirements for vulnerabilit...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>There are several new requirements for vulnera...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>19</th>\n",
+       "      <td>orca-2-7b</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>more on penetration testing</td>\n",
+       "      <td>What are some key differences between penetrat...</td>\n",
+       "      <td>[PCI DSS v4.0 ROC Template r1  December 2022  ...</td>\n",
+       "      <td>Penetration testing and vulnerability assessme...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>20</th>\n",
+       "      <td>orca-2-7b</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>What's PCI DSS?</td>\n",
+       "      <td></td>\n",
+       "      <td>[PCI Data Security Standard\\nPCI DSS provides ...</td>\n",
+       "      <td>PCI DSS stands for Payment Card Industry Data ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>21</th>\n",
+       "      <td>orca-2-7b</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>Can you summarize the changes made from PCI DS...</td>\n",
+       "      <td>How does PCI DSS version 4.0 differ from versi...</td>\n",
+       "      <td>[Payment Card Industry  \\nData Security Standa...</td>\n",
+       "      <td>In terms of security standards, PCI DSS versio...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>22</th>\n",
+       "      <td>orca-2-7b</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>new requirements for vulnerability assessments</td>\n",
+       "      <td>What are the new requirements for vulnerabilit...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>There are several new requirements related to ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>23</th>\n",
+       "      <td>orca-2-7b</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>more on penetration testing</td>\n",
+       "      <td>What are some specific techniques used during ...</td>\n",
+       "      <td>[circumvent or defeat the security features of...</td>\n",
+       "      <td>Some specific techniques used during penetrati...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>24</th>\n",
+       "      <td>orca-2-7b_tgi</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>What's PCI DSS?</td>\n",
+       "      <td></td>\n",
+       "      <td>[PCI Data Security Standard\\nPCI DSS provides ...</td>\n",
+       "      <td>PCI DSS stands for Payment Card Industry Data...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>25</th>\n",
+       "      <td>orca-2-7b_tgi</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>Can you summarize the changes made from PCI DS...</td>\n",
+       "      <td>What are the key differences between PCI DSS ...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>The key differences between PCI DSS version 3...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>26</th>\n",
+       "      <td>orca-2-7b_tgi</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>new requirements for vulnerability assessments</td>\n",
+       "      <td>What are the new requirements for vulnerabili...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>The new requirements for vulnerability assess...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>27</th>\n",
+       "      <td>orca-2-7b_tgi</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>more on penetration testing</td>\n",
+       "      <td>What are the new requirements for penetration...</td>\n",
+       "      <td>[PCI DSS v4.0 ROC Template r1  December 2022  ...</td>\n",
+       "      <td>In PCI DSS version 4.0, the new requirements ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>28</th>\n",
+       "      <td>orca-2-7b_tgi</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>What's PCI DSS?</td>\n",
+       "      <td></td>\n",
+       "      <td>[PCI Data Security Standard\\nPCI DSS provides ...</td>\n",
+       "      <td>PCI DSS stands for Payment Card Industry Data...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>29</th>\n",
+       "      <td>orca-2-7b_tgi</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>Can you summarize the changes made from PCI DS...</td>\n",
+       "      <td>What are the key differences between PCI DSS ...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>The key differences between PCI DSS versions ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>30</th>\n",
+       "      <td>orca-2-7b_tgi</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>new requirements for vulnerability assessments</td>\n",
+       "      <td>What are the new requirements for vulnerabili...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>There are several new requirements related to...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>31</th>\n",
+       "      <td>orca-2-7b_tgi</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>more on penetration testing</td>\n",
+       "      <td>What are some common methods used in penetrat...</td>\n",
+       "      <td>[PCI DSS v4.0 ROC Template r1  December 2022  ...</td>\n",
+       "      <td>Some common methods used in penetration testi...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>32</th>\n",
+       "      <td>orca-2-7b_tgi</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>What's PCI DSS?</td>\n",
+       "      <td></td>\n",
+       "      <td>[PCI Data Security Standard\\nPCI DSS provides ...</td>\n",
+       "      <td>PCI DSS stands for Payment Card Industry Data...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>33</th>\n",
+       "      <td>orca-2-7b_tgi</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>Can you summarize the changes made from PCI DS...</td>\n",
+       "      <td>Summarize the differences between PCI DSS ver...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>The main difference between PCI DSS version 3...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>34</th>\n",
+       "      <td>orca-2-7b_tgi</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>new requirements for vulnerability assessments</td>\n",
+       "      <td>What are the new requirements for vulnerabili...</td>\n",
+       "      <td>[PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...</td>\n",
+       "      <td>There are no specific changes mentioned for v...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>35</th>\n",
+       "      <td>orca-2-7b_tgi</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>more on penetration testing</td>\n",
+       "      <td>What are some best practices for conducting e...</td>\n",
+       "      <td>[methodology  examined for this testing \\nproc...</td>\n",
+       "      <td>Some best practices for conducting effective ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>36</th>\n",
+       "      <td>orca-2-13b_tgi</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>What's PCI DSS?</td>\n",
+       "      <td></td>\n",
+       "      <td>[PCI Data Security Standard\\nPCI DSS provides ...</td>\n",
+       "      <td>PCI DSS is a global standard that provides a ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>37</th>\n",
+       "      <td>orca-2-13b_tgi</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>Can you summarize the changes made from PCI DS...</td>\n",
+       "      <td>¿Puedes resumir los cambios realizados desde ...</td>\n",
+       "      <td>[April 2015  PCI DSS 3.1, \\nRevision  1.0 Revi...</td>\n",
+       "      <td>Sí, puedo resumir los cambios realizados desd...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>38</th>\n",
+       "      <td>orca-2-13b_tgi</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>new requirements for vulnerability assessments</td>\n",
+       "      <td>¿Cuáles son las nuevas recomendaciones para l...</td>\n",
+       "      <td>[The Prioritized Approach to Pursue PCI DSS Co...</td>\n",
+       "      <td>Las nuevas recomendaciones para las evaluacion...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>39</th>\n",
+       "      <td>orca-2-13b_tgi</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>more on penetration testing</td>\n",
+       "      <td>¿Puedes dar más información sobre las pruebas...</td>\n",
+       "      <td>[methodology  examined for this testing \\nproc...</td>\n",
+       "      <td>Sí, puedo dar más información sobre las prueb...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>40</th>\n",
+       "      <td>orca-2-13b_tgi</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>What's PCI DSS?</td>\n",
+       "      <td></td>\n",
+       "      <td>[PCI Data Security Standard\\nPCI DSS provides ...</td>\n",
+       "      <td>PCI DSS is a set of requirements designed to ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>41</th>\n",
+       "      <td>orca-2-13b_tgi</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>Can you summarize the changes made from PCI DS...</td>\n",
+       "      <td>¿Puedes resumir los cambios realizados desde ...</td>\n",
+       "      <td>[April 2015  PCI DSS 3.1, \\nRevision  1.0 Revi...</td>\n",
+       "      <td>Sí, puedo resumir los cambios realizados desd...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>42</th>\n",
+       "      <td>orca-2-13b_tgi</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>new requirements for vulnerability assessments</td>\n",
+       "      <td>¿Cuáles son las nuevas exigencias para las ev...</td>\n",
+       "      <td>[The Prioritized Approach to Pursue PCI DSS Co...</td>\n",
+       "      <td>Para obtener información sobre las nuevas exi...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>43</th>\n",
+       "      <td>orca-2-13b_tgi</td>\n",
+       "      <td>1.10</td>\n",
+       "      <td>more on penetration testing</td>\n",
+       "      <td>¿Podría proporcionar más información sobre la...</td>\n",
+       "      <td>[The Prioritized Approach to Pursue PCI DSS Co...</td>\n",
+       "      <td>Sí, puedo proporcionar más información sobre ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>44</th>\n",
+       "      <td>orca-2-13b_tgi</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>What's PCI DSS?</td>\n",
+       "      <td></td>\n",
+       "      <td>[PCI Data Security Standard\\nPCI DSS provides ...</td>\n",
+       "      <td>PCI DSS is a set of requirements designed to ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>45</th>\n",
+       "      <td>orca-2-13b_tgi</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>Can you summarize the changes made from PCI DS...</td>\n",
+       "      <td>¿Puedes resumir los cambios realizados desde ...</td>\n",
+       "      <td>[April 2015  PCI DSS 3.1, \\nRevision  1.0 Revi...</td>\n",
+       "      <td>Sí, puedo resumir los cambios realizados desd...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>46</th>\n",
+       "      <td>orca-2-13b_tgi</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>new requirements for vulnerability assessments</td>\n",
+       "      <td>¿Cuáles son las nuevas recomendaciones para l...</td>\n",
+       "      <td>[The Prioritized Approach to Pursue PCI DSS Co...</td>\n",
+       "      <td>Some possible helpful answers are:\\n\\n- Las n...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>47</th>\n",
+       "      <td>orca-2-13b_tgi</td>\n",
+       "      <td>1.15</td>\n",
+       "      <td>more on penetration testing</td>\n",
+       "      <td>¿Cuál es el papel del prueba de penetración e...</td>\n",
+       "      <td>[The Prioritized Approach to Pursue PCI DSS Co...</td>\n",
+       "      <td>La prueba de penetración es una parte importa...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "        model_name repetition_penalty  \\\n",
+       "0       llama-2-7b               1.05   \n",
+       "1       llama-2-7b               1.05   \n",
+       "2       llama-2-7b               1.05   \n",
+       "3       llama-2-7b               1.05   \n",
+       "4       llama-2-7b               1.10   \n",
+       "5       llama-2-7b               1.10   \n",
+       "6       llama-2-7b               1.10   \n",
+       "7       llama-2-7b               1.10   \n",
+       "8       llama-2-7b               1.15   \n",
+       "9       llama-2-7b               1.15   \n",
+       "10      llama-2-7b               1.15   \n",
+       "11      llama-2-7b               1.15   \n",
+       "12       orca-2-7b               1.05   \n",
+       "13       orca-2-7b               1.05   \n",
+       "14       orca-2-7b               1.05   \n",
+       "15       orca-2-7b               1.05   \n",
+       "16       orca-2-7b               1.10   \n",
+       "17       orca-2-7b               1.10   \n",
+       "18       orca-2-7b               1.10   \n",
+       "19       orca-2-7b               1.10   \n",
+       "20       orca-2-7b               1.15   \n",
+       "21       orca-2-7b               1.15   \n",
+       "22       orca-2-7b               1.15   \n",
+       "23       orca-2-7b               1.15   \n",
+       "24   orca-2-7b_tgi               1.05   \n",
+       "25   orca-2-7b_tgi               1.05   \n",
+       "26   orca-2-7b_tgi               1.05   \n",
+       "27   orca-2-7b_tgi               1.05   \n",
+       "28   orca-2-7b_tgi               1.10   \n",
+       "29   orca-2-7b_tgi               1.10   \n",
+       "30   orca-2-7b_tgi               1.10   \n",
+       "31   orca-2-7b_tgi               1.10   \n",
+       "32   orca-2-7b_tgi               1.15   \n",
+       "33   orca-2-7b_tgi               1.15   \n",
+       "34   orca-2-7b_tgi               1.15   \n",
+       "35   orca-2-7b_tgi               1.15   \n",
+       "36  orca-2-13b_tgi               1.05   \n",
+       "37  orca-2-13b_tgi               1.05   \n",
+       "38  orca-2-13b_tgi               1.05   \n",
+       "39  orca-2-13b_tgi               1.05   \n",
+       "40  orca-2-13b_tgi               1.10   \n",
+       "41  orca-2-13b_tgi               1.10   \n",
+       "42  orca-2-13b_tgi               1.10   \n",
+       "43  orca-2-13b_tgi               1.10   \n",
+       "44  orca-2-13b_tgi               1.15   \n",
+       "45  orca-2-13b_tgi               1.15   \n",
+       "46  orca-2-13b_tgi               1.15   \n",
+       "47  orca-2-13b_tgi               1.15   \n",
+       "\n",
+       "                                        user_question  \\\n",
+       "0                                     What's PCI DSS?   \n",
+       "1   Can you summarize the changes made from PCI DS...   \n",
+       "2      new requirements for vulnerability assessments   \n",
+       "3                         more on penetration testing   \n",
+       "4                                     What's PCI DSS?   \n",
+       "5   Can you summarize the changes made from PCI DS...   \n",
+       "6      new requirements for vulnerability assessments   \n",
+       "7                         more on penetration testing   \n",
+       "8                                     What's PCI DSS?   \n",
+       "9   Can you summarize the changes made from PCI DS...   \n",
+       "10     new requirements for vulnerability assessments   \n",
+       "11                        more on penetration testing   \n",
+       "12                                    What's PCI DSS?   \n",
+       "13  Can you summarize the changes made from PCI DS...   \n",
+       "14     new requirements for vulnerability assessments   \n",
+       "15                        more on penetration testing   \n",
+       "16                                    What's PCI DSS?   \n",
+       "17  Can you summarize the changes made from PCI DS...   \n",
+       "18     new requirements for vulnerability assessments   \n",
+       "19                        more on penetration testing   \n",
+       "20                                    What's PCI DSS?   \n",
+       "21  Can you summarize the changes made from PCI DS...   \n",
+       "22     new requirements for vulnerability assessments   \n",
+       "23                        more on penetration testing   \n",
+       "24                                    What's PCI DSS?   \n",
+       "25  Can you summarize the changes made from PCI DS...   \n",
+       "26     new requirements for vulnerability assessments   \n",
+       "27                        more on penetration testing   \n",
+       "28                                    What's PCI DSS?   \n",
+       "29  Can you summarize the changes made from PCI DS...   \n",
+       "30     new requirements for vulnerability assessments   \n",
+       "31                        more on penetration testing   \n",
+       "32                                    What's PCI DSS?   \n",
+       "33  Can you summarize the changes made from PCI DS...   \n",
+       "34     new requirements for vulnerability assessments   \n",
+       "35                        more on penetration testing   \n",
+       "36                                    What's PCI DSS?   \n",
+       "37  Can you summarize the changes made from PCI DS...   \n",
+       "38     new requirements for vulnerability assessments   \n",
+       "39                        more on penetration testing   \n",
+       "40                                    What's PCI DSS?   \n",
+       "41  Can you summarize the changes made from PCI DS...   \n",
+       "42     new requirements for vulnerability assessments   \n",
+       "43                        more on penetration testing   \n",
+       "44                                    What's PCI DSS?   \n",
+       "45  Can you summarize the changes made from PCI DS...   \n",
+       "46     new requirements for vulnerability assessments   \n",
+       "47                        more on penetration testing   \n",
+       "\n",
+       "                                  standalone_question  \\\n",
+       "0                                                       \n",
+       "1   What are the key changes made in PCI DSS versi...   \n",
+       "2   What are the updated requirements for vulnerab...   \n",
+       "3   What are the updated requirements for vulnerab...   \n",
+       "4                                                       \n",
+       "5   What are the key changes made in PCI DSS versi...   \n",
+       "6   What are the new requirements for vulnerabilit...   \n",
+       "7   What are the key differences between penetrati...   \n",
+       "8                                                       \n",
+       "9   What are the key changes between PCI DSS versi...   \n",
+       "10  What are some of the new requirements for vuln...   \n",
+       "11  What are some common techniques used in penetr...   \n",
+       "12                                                      \n",
+       "13  What are the main differences between PCI DSS ...   \n",
+       "14  What are the new requirements for vulnerabilit...   \n",
+       "15  What are the new requirements for penetration ...   \n",
+       "16                                                      \n",
+       "17  What are the main differences between PCI DSS ...   \n",
+       "18  What are the new requirements for vulnerabilit...   \n",
+       "19  What are some key differences between penetrat...   \n",
+       "20                                                      \n",
+       "21  How does PCI DSS version 4.0 differ from versi...   \n",
+       "22  What are the new requirements for vulnerabilit...   \n",
+       "23  What are some specific techniques used during ...   \n",
+       "24                                                      \n",
+       "25   What are the key differences between PCI DSS ...   \n",
+       "26   What are the new requirements for vulnerabili...   \n",
+       "27   What are the new requirements for penetration...   \n",
+       "28                                                      \n",
+       "29   What are the key differences between PCI DSS ...   \n",
+       "30   What are the new requirements for vulnerabili...   \n",
+       "31   What are some common methods used in penetrat...   \n",
+       "32                                                      \n",
+       "33   Summarize the differences between PCI DSS ver...   \n",
+       "34   What are the new requirements for vulnerabili...   \n",
+       "35   What are some best practices for conducting e...   \n",
+       "36                                                      \n",
+       "37   ¿Puedes resumir los cambios realizados desde ...   \n",
+       "38   ¿Cuáles son las nuevas recomendaciones para l...   \n",
+       "39   ¿Puedes dar más información sobre las pruebas...   \n",
+       "40                                                      \n",
+       "41   ¿Puedes resumir los cambios realizados desde ...   \n",
+       "42   ¿Cuáles son las nuevas exigencias para las ev...   \n",
+       "43   ¿Podría proporcionar más información sobre la...   \n",
+       "44                                                      \n",
+       "45   ¿Puedes resumir los cambios realizados desde ...   \n",
+       "46   ¿Cuáles son las nuevas recomendaciones para l...   \n",
+       "47   ¿Cuál es el papel del prueba de penetración e...   \n",
+       "\n",
+       "                                             contexts  \\\n",
+       "0   [PCI Data Security Standard\\nPCI DSS provides ...   \n",
+       "1   [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "2   [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "3   [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "4   [PCI Data Security Standard\\nPCI DSS provides ...   \n",
+       "5   [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "6   [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "7   [PCI DSS v4.0 ROC Template r1  December 2022  ...   \n",
+       "8   [PCI Data Security Standard\\nPCI DSS provides ...   \n",
+       "9   [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "10  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "11  [circumvent or defeat the security features of...   \n",
+       "12  [PCI Data Security Standard\\nPCI DSS provides ...   \n",
+       "13  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "14  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "15  [PCI DSS v4.0 ROC Template r1  December 2022  ...   \n",
+       "16  [PCI Data Security Standard\\nPCI DSS provides ...   \n",
+       "17  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "18  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "19  [PCI DSS v4.0 ROC Template r1  December 2022  ...   \n",
+       "20  [PCI Data Security Standard\\nPCI DSS provides ...   \n",
+       "21  [Payment Card Industry  \\nData Security Standa...   \n",
+       "22  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "23  [circumvent or defeat the security features of...   \n",
+       "24  [PCI Data Security Standard\\nPCI DSS provides ...   \n",
+       "25  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "26  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "27  [PCI DSS v4.0 ROC Template r1  December 2022  ...   \n",
+       "28  [PCI Data Security Standard\\nPCI DSS provides ...   \n",
+       "29  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "30  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "31  [PCI DSS v4.0 ROC Template r1  December 2022  ...   \n",
+       "32  [PCI Data Security Standard\\nPCI DSS provides ...   \n",
+       "33  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "34  [PCI DSS v3.2.1 to v.4.0 Summary of Changes  r...   \n",
+       "35  [methodology  examined for this testing \\nproc...   \n",
+       "36  [PCI Data Security Standard\\nPCI DSS provides ...   \n",
+       "37  [April 2015  PCI DSS 3.1, \\nRevision  1.0 Revi...   \n",
+       "38  [The Prioritized Approach to Pursue PCI DSS Co...   \n",
+       "39  [methodology  examined for this testing \\nproc...   \n",
+       "40  [PCI Data Security Standard\\nPCI DSS provides ...   \n",
+       "41  [April 2015  PCI DSS 3.1, \\nRevision  1.0 Revi...   \n",
+       "42  [The Prioritized Approach to Pursue PCI DSS Co...   \n",
+       "43  [The Prioritized Approach to Pursue PCI DSS Co...   \n",
+       "44  [PCI Data Security Standard\\nPCI DSS provides ...   \n",
+       "45  [April 2015  PCI DSS 3.1, \\nRevision  1.0 Revi...   \n",
+       "46  [The Prioritized Approach to Pursue PCI DSS Co...   \n",
+       "47  [The Prioritized Approach to Pursue PCI DSS Co...   \n",
+       "\n",
+       "                                               answer  \n",
+       "0   PCI DSS stands for Payment Card Industry Data ...  \n",
+       "1    The key changes made in PCI DSS version 4.0 c...  \n",
+       "2    The updated requirements for vulnerability as...  \n",
+       "3    The updated requirements for vulnerability as...  \n",
+       "4   PCI DSS stands for Payment Card Industry Data ...  \n",
+       "5    The key changes made in PCI DSS version 4.0 c...  \n",
+       "6    According to the provided summary of changes,...  \n",
+       "7   Penetration testing and vulnerability assessme...  \n",
+       "8   PCI DSS stands for Payment Card Industry Data ...  \n",
+       "9   The key changes between PCI DSS versions 3.2.1...  \n",
+       "10  In PCI DSS version 4.0, there are several new ...  \n",
+       "11  Penetration testing involves various technique...  \n",
+       "12  PCI DSS stands for Payment Card Industry Data ...  \n",
+       "13  The main differences between PCI DSS version 3...  \n",
+       "14  The new requirements for vulnerability assessm...  \n",
+       "15  The new requirements for penetration testing i...  \n",
+       "16  PCI DSS stands for Payment Card Industry Data ...  \n",
+       "17  The main differences between PCI DSS versions ...  \n",
+       "18  There are several new requirements for vulnera...  \n",
+       "19  Penetration testing and vulnerability assessme...  \n",
+       "20  PCI DSS stands for Payment Card Industry Data ...  \n",
+       "21  In terms of security standards, PCI DSS versio...  \n",
+       "22  There are several new requirements related to ...  \n",
+       "23  Some specific techniques used during penetrati...  \n",
+       "24   PCI DSS stands for Payment Card Industry Data...  \n",
+       "25   The key differences between PCI DSS version 3...  \n",
+       "26   The new requirements for vulnerability assess...  \n",
+       "27   In PCI DSS version 4.0, the new requirements ...  \n",
+       "28   PCI DSS stands for Payment Card Industry Data...  \n",
+       "29   The key differences between PCI DSS versions ...  \n",
+       "30   There are several new requirements related to...  \n",
+       "31   Some common methods used in penetration testi...  \n",
+       "32   PCI DSS stands for Payment Card Industry Data...  \n",
+       "33   The main difference between PCI DSS version 3...  \n",
+       "34   There are no specific changes mentioned for v...  \n",
+       "35   Some best practices for conducting effective ...  \n",
+       "36   PCI DSS is a global standard that provides a ...  \n",
+       "37   Sí, puedo resumir los cambios realizados desd...  \n",
+       "38  Las nuevas recomendaciones para las evaluacion...  \n",
+       "39   Sí, puedo dar más información sobre las prueb...  \n",
+       "40   PCI DSS is a set of requirements designed to ...  \n",
+       "41   Sí, puedo resumir los cambios realizados desd...  \n",
+       "42   Para obtener información sobre las nuevas exi...  \n",
+       "43   Sí, puedo proporcionar más información sobre ...  \n",
+       "44   PCI DSS is a set of requirements designed to ...  \n",
+       "45   Sí, puedo resumir los cambios realizados desd...  \n",
+       "46   Some possible helpful answers are:\\n\\n- Las n...  \n",
+       "47   La prueba de penetración es una parte importa...  "
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "raw_pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "raw_pd.to_excel(\"./results/raw_data_nvidia_4080.xlsx\", index=False)\n",
+    "perf_pd.to_excel(\"./results/perf_data_nvidia_4080.xlsx\", index=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "('new requirements for vulnerability assessments',\n",
+       " 'What are the updated requirements for vulnerability assessments in PCI DSS version 4.0 compared to version 3.2.1?\\n\\n\\n\\n',\n",
+       " ' The updated requirements for vulnerability assessments in PCI DSS version 4.0 are as follows:\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n')"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "raw_pd[\"user_question\"][2], raw_pd[\"standalone_question\"][2], raw_pd[\"answer\"][2]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "('new requirements for vulnerability assessments',\n",
+       " 'What are the new requirements for vulnerability assessments in PCI DSS version 4.0?\\n\\n\\n\\n',\n",
+       " 'The new requirements for vulnerability assessments in PCI DSS version 4.0 include:\\n\\n- Requirement 4.1.2, which states that there is a new requirement for roles and responsibilities related to vulnerability assessments.\\n- Requirement 4.2.1, which requires confirmation that certificates used for Point-to-Point (PAN) transmissions over open, public networks are valid and not expired or revoked. This bullet is a best practice until March 31, 2025.\\n- Requirement 4.2.1.1, which introduces a new requirement to maintain an inventory of trusted keys and certificates. This is also a best practice until March 31, 2025.\\n\\n\\n\\n')"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "raw_pd[\"user_question\"][14], raw_pd[\"standalone_question\"][14], raw_pd[\"answer\"][14]"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

results/4_translate_spanish_content.ipynb

@@ -0,0 +1,465 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "workding dir: /Users/inflaton/code/engd/papers/chat-with-orca-2\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import sys\n",
+    "from pathlib import Path\n",
+    "\n",
+    "workding_dir = str(Path.cwd().parent)\n",
+    "os.chdir(workding_dir)\n",
+    "sys.path.append(workding_dir)\n",
+    "print(\"workding dir:\", workding_dir)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Translating Orca-2-13b's Results from Spanish to English"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>model_name</th>\n",
+       "      <th>repetition_penalty</th>\n",
+       "      <th>faithfulness</th>\n",
+       "      <th>answer_relevancy</th>\n",
+       "      <th>overall_score</th>\n",
+       "      <th>total_time_used</th>\n",
+       "      <th>num_tokens_generated</th>\n",
+       "      <th>token_per_second</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>orca-2-13b</td>\n",
+       "      <td>1.05</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>0.987592</td>\n",
+       "      <td>0.993757</td>\n",
+       "      <td>397.548</td>\n",
+       "      <td>641</td>\n",
+       "      <td>1.612</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   model_name  repetition_penalty  faithfulness  answer_relevancy  \\\n",
+       "6  orca-2-13b                1.05           1.0          0.987592   \n",
+       "\n",
+       "   overall_score  total_time_used  num_tokens_generated  token_per_second  \n",
+       "6       0.993757          397.548                   641             1.612  "
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "\n",
+    "perf_pd1 = pd.read_excel(\"./results/perf_data.xlsx\")\n",
+    "idx = (\n",
+    "    perf_pd1[perf_pd1[\"model_name\"] == \"orca-2-13b\"]\n",
+    "    .groupby(\"model_name\")[\"overall_score\"]\n",
+    "    .idxmax()\n",
+    ")\n",
+    "df = perf_pd1.loc[idx].sort_index(ascending=False)\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "selected = []\n",
+    "NUM_QUESTIONS =4\n",
+    "for j in range(NUM_QUESTIONS):\n",
+    "    for i in idx.values:\n",
+    "        selected.append(i * NUM_QUESTIONS + j)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Model Name</th>\n",
+       "      <th>LLM Generated Content</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>orca-2-13b</td>\n",
+       "      <td>PCI DSS is a global standard that provides a b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>orca-2-13b</td>\n",
+       "      <td>**¿Puedes resumir los cambios realizados desde...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>orca-2-13b</td>\n",
+       "      <td>**¿Cuáles son las nuevas requisitos para las e...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>orca-2-13b</td>\n",
+       "      <td>**¿Puedes dar más detalles sobre las prácticas...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   Model Name                              LLM Generated Content\n",
+       "0  orca-2-13b  PCI DSS is a global standard that provides a b...\n",
+       "1  orca-2-13b  **¿Puedes resumir los cambios realizados desde...\n",
+       "2  orca-2-13b  **¿Cuáles son las nuevas requisitos para las e...\n",
+       "3  orca-2-13b  **¿Puedes dar más detalles sobre las prácticas..."
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "raw_data = pd.read_excel(\"./results/raw_data.xlsx\")\n",
+    "df = raw_data.loc[selected]\n",
+    "df = df.drop([\"repetition_penalty\", \"contexts\"], axis=1)\n",
+    "df = df.fillna(\"\")\n",
+    "df[df.columns] = df.apply(lambda x: x.str.strip())\n",
+    "df[\"standalone_question\"] = df[\"standalone_question\"].str.replace(\"\\n\", \"**\\n**\")\n",
+    "df[\"standalone_question\"] = df[\"standalone_question\"].apply(\n",
+    "    lambda x: \"{}{}{}\".format(\"**\", x, \"**\") if len(x) > 0 else x\n",
+    ")\n",
+    "df[\"standalone_question\"] = df[\"standalone_question\"].str.replace(\"****\", \"\")\n",
+    "df[\"LLM Generated Content\"] = (\n",
+    "    df[\"standalone_question\"].str.cat(df[\"answer\"], sep=\"\\n\").str.strip()\n",
+    ")\n",
+    "df = df.rename(columns={\"model_name\": \"Model Name\"})\n",
+    "df = df.drop(columns=[\"answer\", \"standalone_question\", \"user_question\"])\n",
+    "df[df.columns] = df.apply(lambda x: x.str.strip())\n",
+    "df.reset_index(drop=True, inplace=True)\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Model Name</th>\n",
+       "      <th>LLM Generated Content</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>orca-2-13b</td>\n",
+       "      <td>¿Puedes resumir los cambios realizados desde l...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>orca-2-13b</td>\n",
+       "      <td>¿Cuáles son las nuevas requisitos para las eva...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>orca-2-13b</td>\n",
+       "      <td>¿Puedes dar más detalles sobre las prácticas d...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   Model Name                              LLM Generated Content\n",
+       "1  orca-2-13b  ¿Puedes resumir los cambios realizados desde l...\n",
+       "2  orca-2-13b  ¿Cuáles son las nuevas requisitos para las eva...\n",
+       "3  orca-2-13b  ¿Puedes dar más detalles sobre las prácticas d..."
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.drop(index=[0], inplace=True)\n",
+    "orca_2_13b_df = df\n",
+    "orca_2_13b_df[\"LLM Generated Content\"] = orca_2_13b_df[\"LLM Generated Content\"].apply(\n",
+    "    lambda x: x.replace(\"**\", \"\")\n",
+    ")\n",
+    "orca_2_13b_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 44,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "from openai import OpenAI\n",
+    "from app_modules.init import *\n",
+    "\n",
+    "\n",
+    "# Function to translate text using OpenAI\n",
+    "def translate(text):\n",
+    "    client = OpenAI(\n",
+    "        # This is the default and can be omitted\n",
+    "        api_key=os.environ.get(\"OPENAI_API_KEY\"),\n",
+    "    )\n",
+    "\n",
+    "    chat_completion = client.chat.completions.create(\n",
+    "        messages=[\n",
+    "            {\n",
+    "                \"role\": \"user\",\n",
+    "                \"content\": f\"Translate the following text to English: {text}\",\n",
+    "            }\n",
+    "        ],\n",
+    "        model=\"gpt-4-turbo-preview\",\n",
+    "    )\n",
+    "    print(chat_completion)\n",
+    "    return chat_completion.choices[0].message.content"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 58,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "ChatCompletion(id='chatcmpl-8mdHUCObrRgeCDBweZXGDc0FTVrh8', choices=[Choice(finish_reason='stop', index=0, logprobs=None, message=ChatCompletionMessage(content='Can you summarize the changes made from version 3.2.1 to version 4.0 of PCI DSS?\\nYes, I can summarize the changes made from version 3.2.1 to version 4.0 of PCI DSS. The changes include adjustments to align with the modifications from PCI DSS 3.2 to PCI DSS 3.2.1, as well as minor corrections and edits to clarify and improve the format. Then, in version 4.0, updates were made to align with the modifications from PCI DSS 3.2.1 to PCI DSS 4.0, and there were also minor corrections and edits made to clarify and improve the format. In December 2022, there was an additional revision that included minor adjustments, correction of typographical errors, and the removal of \"In Place with Remediation\" as a reporting option.', role='assistant', function_call=None, tool_calls=None))], created=1706601432, model='gpt-4-0125-preview', object='chat.completion', system_fingerprint='fp_376b7f78b9', usage=CompletionUsage(completion_tokens=184, prompt_tokens=240, total_tokens=424))\n",
+      "ChatCompletion(id='chatcmpl-8mdHpGDnxgKsm6YBRSw8irwVpMz0y', choices=[Choice(finish_reason='stop', index=0, logprobs=None, message=ChatCompletionMessage(content='What are the new requirements for vulnerability assessments in PCI DSS version 4.0?\\nThe new requirements for vulnerability assessments in PCI DSS version 4.0 include resolving vulnerabilities with a CVSS score of 4.0 or higher, performing rescans when necessary, and having a defined, documented, and implemented penetration testing method that includes industry-accepted approaches and coverage for the CDE perimeter and critical systems.', role='assistant', function_call=None, tool_calls=None))], created=1706601453, model='gpt-4-0125-preview', object='chat.completion', system_fingerprint='fp_376b7f78b9', usage=CompletionUsage(completion_tokens=84, prompt_tokens=140, total_tokens=224))\n",
+      "ChatCompletion(id='chatcmpl-8mdHv2yENauX11654tNa99RW1dMHo', choices=[Choice(finish_reason='stop', index=0, logprobs=None, message=ChatCompletionMessage(content='Can you provide more details about penetration testing practices in PCI DSS?\\nYes, I can provide more details about penetration testing practices in PCI DSS. The PCI DSS requires that external penetration testing be conducted at least once every 12 months and after any significant modification or change in the infrastructure or software. The penetration testing must be conducted by a qualified internal resource or a qualified third party from outside the organization and must comply with the methodology defined by the entity. Additionally, the security vulnerabilities and weaknesses that are found during the penetration testing must be corrected and verified by repeating the testing.', role='assistant', function_call=None, tool_calls=None))], created=1706601459, model='gpt-4-0125-preview', object='chat.completion', system_fingerprint='fp_376b7f78b9', usage=CompletionUsage(completion_tokens=116, prompt_tokens=177, total_tokens=293))\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Model Name</th>\n",
+       "      <th>LLM Generated Content</th>\n",
+       "      <th>Translation by GPT-4-Turbo</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>orca-2-13b</td>\n",
+       "      <td>¿Puedes resumir los cambios realizados desde l...</td>\n",
+       "      <td>Can you summarize the changes made from versio...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>orca-2-13b</td>\n",
+       "      <td>¿Cuáles son las nuevas requisitos para las eva...</td>\n",
+       "      <td>What are the new requirements for vulnerabilit...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>orca-2-13b</td>\n",
+       "      <td>¿Puedes dar más detalles sobre las prácticas d...</td>\n",
+       "      <td>Can you provide more details about penetration...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   Model Name                              LLM Generated Content  \\\n",
+       "1  orca-2-13b  ¿Puedes resumir los cambios realizados desde l...   \n",
+       "2  orca-2-13b  ¿Cuáles son las nuevas requisitos para las eva...   \n",
+       "3  orca-2-13b  ¿Puedes dar más detalles sobre las prácticas d...   \n",
+       "\n",
+       "                          Translation by GPT-4-Turbo  \n",
+       "1  Can you summarize the changes made from versio...  \n",
+       "2  What are the new requirements for vulnerabilit...  \n",
+       "3  Can you provide more details about penetration...  "
+      ]
+     },
+     "execution_count": 58,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "orca_2_13b_df[\"Translation by GPT-4-Turbo\"] = orca_2_13b_df[\n",
+    "    \"LLM Generated Content\"\n",
+    "].apply(translate)\n",
+    "orca_2_13b_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 59,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "|    | Model Name   | LLM Generated Content                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         | Translation by GPT-4-Turbo                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |\n",
+      "|---:|:-------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|\n",
+      "|  1 | orca-2-13b   | ¿Puedes resumir los cambios realizados desde la versión 3.2.1 hasta la versión 4.0 de PCI DSS?                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                | Can you summarize the changes made from version 3.2.1 to version 4.0 of PCI DSS?                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |\n",
+      "|    |              | Sí, puedo resumir los cambios realizados desde la versión 3.2.1 hasta la versión 4.0 de PCI DSS. Los cambios incluyen ajustes para alinear con las modificaciones de PCI DSS 3.2 a PCI DSS 3.2.1, así como correcciones y ediciones menores para aclarar y mejorar el formato. Luego, en la versión 4.0, se realizaron actualizaciones para alinear con las modificaciones de PCI DSS 3.2.1 a PCI DSS 4.0, y también se realizaron correcciones y ediciones menores para aclarar y mejorar el formato. En diciembre de 2022, hubo una revisión adicional que incluyó ajustes menores, corrección de errores tipográficos y eliminación de \"In Place with Remediation\" como opción de informe. | Yes, I can summarize the changes made from version 3.2.1 to version 4.0 of PCI DSS. The changes include adjustments to align with the modifications from PCI DSS 3.2 to PCI DSS 3.2.1, as well as minor corrections and edits to clarify and improve the format. Then, in version 4.0, updates were made to align with the modifications from PCI DSS 3.2.1 to PCI DSS 4.0, and there were also minor corrections and edits made to clarify and improve the format. In December 2022, there was an additional revision that included minor adjustments, correction of typographical errors, and the removal of \"In Place with Remediation\" as a reporting option. |\n",
+      "|  2 | orca-2-13b   | ¿Cuáles son las nuevas requisitos para las evaluaciones de vulnerabilidades en la versión 4.0 de PCI DSS?                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     | What are the new requirements for vulnerability assessments in PCI DSS version 4.0?                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |\n",
+      "|    |              | Las nuevas requisitos para las evaluaciones de vulnerabilidades en la versión 4.0 de PCI DSS incluyen resolver vulnerabilidades con una puntuación de 4.0 o más en el CVSS, realizar rescans cuando sea necesario, y tener un método de penetration testing definido, documentado e implementado que incluye enfoques aceptados por la industria y cobertura para el perímetro del SDE y los sistemas críticos.                                                                                                                                                                                                                                                                               | The new requirements for vulnerability assessments in PCI DSS version 4.0 include resolving vulnerabilities with a CVSS score of 4.0 or higher, performing rescans when necessary, and having a defined, documented, and implemented penetration testing method that includes industry-accepted approaches and coverage for the CDE perimeter and critical systems.                                                                                                                                                                                                                                                                                               |\n",
+      "|  3 | orca-2-13b   | ¿Puedes dar más detalles sobre las prácticas de prueba de penetración en PCI DSS?                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             | Can you provide more details about penetration testing practices in PCI DSS?                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |\n",
+      "|    |              | Sí, puedo dar más detalles sobre las prácticas de prueba de penetración en PCI DSS. La PCI DSS requiere que se realice un penetration testing externo al menos una vez cada 12 meses y después de cualquier modificación o cambio significativo en la infraestructura o el software. El penetration testing debe ser realizado por un recurso interno calificado o un tercero de fuera de la organización calificado y debe cumplir con la metodología definida por la entidad. Además, las vulnerabilidades y debilidades de seguridad que se encuentren durante el penetration testing deben ser corregidas y verificadas mediante la repetición del testing.                               | Yes, I can provide more details about penetration testing practices in PCI DSS. The PCI DSS requires that external penetration testing be conducted at least once every 12 months and after any significant modification or change in the infrastructure or software. The penetration testing must be conducted by a qualified internal resource or a qualified third party from outside the organization and must comply with the methodology defined by the entity. Additionally, the security vulnerabilities and weaknesses that are found during the penetration testing must be corrected and verified by repeating the testing.                            |\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(orca_2_13b_df.to_markdown())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Translation of Contents Generated by Orca-2-13b\n",
+    "\n",
+    "|    | Model Name   | LLM Generated Content                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         | Translation by GPT-4-Turbo                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |\n",
+    "|---:|:-------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|\n",
+    "|  1 | orca-2-13b   | ¿Puedes resumir los cambios realizados desde la versión 3.2.1 hasta la versión 4.0 de PCI DSS?                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                | Can you summarize the changes made from version 3.2.1 to version 4.0 of PCI DSS?                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |\n",
+    "|    |              | Sí, puedo resumir los cambios realizados desde la versión 3.2.1 hasta la versión 4.0 de PCI DSS. Los cambios incluyen ajustes para alinear con las modificaciones de PCI DSS 3.2 a PCI DSS 3.2.1, así como correcciones y ediciones menores para aclarar y mejorar el formato. Luego, en la versión 4.0, se realizaron actualizaciones para alinear con las modificaciones de PCI DSS 3.2.1 a PCI DSS 4.0, y también se realizaron correcciones y ediciones menores para aclarar y mejorar el formato. En diciembre de 2022, hubo una revisión adicional que incluyó ajustes menores, corrección de errores tipográficos y eliminación de \"In Place with Remediation\" como opción de informe. | Yes, I can summarize the changes made from version 3.2.1 to version 4.0 of PCI DSS. The changes include adjustments to align with the modifications from PCI DSS 3.2 to PCI DSS 3.2.1, as well as minor corrections and edits to clarify and improve the format. Then, in version 4.0, updates were made to align with the modifications from PCI DSS 3.2.1 to PCI DSS 4.0, and there were also minor corrections and edits made to clarify and improve the format. In December 2022, there was an additional revision that included minor adjustments, correction of typographical errors, and the removal of \"In Place with Remediation\" as a reporting option. |\n",
+    "|  2 | orca-2-13b   | ¿Cuáles son las nuevas requisitos para las evaluaciones de vulnerabilidades en la versión 4.0 de PCI DSS?                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     | What are the new requirements for vulnerability assessments in PCI DSS version 4.0?                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |\n",
+    "|    |              | Las nuevas requisitos para las evaluaciones de vulnerabilidades en la versión 4.0 de PCI DSS incluyen resolver vulnerabilidades con una puntuación de 4.0 o más en el CVSS, realizar rescans cuando sea necesario, y tener un método de penetration testing definido, documentado e implementado que incluye enfoques aceptados por la industria y cobertura para el perímetro del SDE y los sistemas críticos.                                                                                                                                                                                                                                                                               | The new requirements for vulnerability assessments in PCI DSS version 4.0 include resolving vulnerabilities with a CVSS score of 4.0 or higher, performing rescans when necessary, and having a defined, documented, and implemented penetration testing method that includes industry-accepted approaches and coverage for the CDE perimeter and critical systems.                                                                                                                                                                                                                                                                                               |\n",
+    "|  3 | orca-2-13b   | ¿Puedes dar más detalles sobre las prácticas de prueba de penetración en PCI DSS?                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             | Can you provide more details about penetration testing practices in PCI DSS?                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |\n",
+    "|    |              | Sí, puedo dar más detalles sobre las prácticas de prueba de penetración en PCI DSS. La PCI DSS requiere que se realice un penetration testing externo al menos una vez cada 12 meses y después de cualquier modificación o cambio significativo en la infraestructura o el software. El penetration testing debe ser realizado por un recurso interno calificado o un tercero de fuera de la organización calificado y debe cumplir con la metodología definida por la entidad. Además, las vulnerabilidades y debilidades de seguridad que se encuentren durante el penetration testing deben ser corregidas y verificadas mediante la repetición del testing.                               | Yes, I can provide more details about penetration testing practices in PCI DSS. The PCI DSS requires that external penetration testing be conducted at least once every 12 months and after any significant modification or change in the infrastructure or software. The penetration testing must be conducted by a qualified internal resource or a qualified third party from outside the organization and must comply with the methodology defined by the entity. Additionally, the security vulnerabilities and weaknesses that are found during the penetration testing must be corrected and verified by repeating the testing.   "
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.18"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}