feat: move gpt out, add confidence level

core-model-prediction/prediction.py

@@ -16,6 +16,8 @@ class PredictRequest(BaseModel):
     backspace_count: int
     typing_duration: int
     letter_click_counts: dict[str, int]
+    gpt35_answer: str
+    gpt4_answer: str
 
 
 class RequestModel(BaseModel):
@@ -39,6 +41,8 @@ def process_instance(data: PredictRequest):
     backspace_count = data.backspace_count
     typing_duration = data.typing_duration
     letter_click_counts = data.letter_click_counts
+    gpt35_answer = data.gpt35_answer
+    gpt4_answer = data.gpt4_answer
 
     # Data preparation for 1st model
     hypothesis = BaseModelHypothesis()
@@ -52,7 +56,8 @@ def process_instance(data: PredictRequest):
     # Data preparation for 2nd model
     secondary_model_dependencies = SecondaryModelDependencies()
     secondary_model_features = secondary_model_dependencies.calculate_features(
-        question, answer, main_model_probability, backspace_count, typing_duration, letter_click_counts)
+        question, answer, main_model_probability, backspace_count, typing_duration,
+        letter_click_counts, gpt35_answer, gpt4_answer)
 
     # 2nd model prediction
     secondary_model = RandomForestModel()
@@ -61,5 +66,16 @@ def process_instance(data: PredictRequest):
 
     return {
         "predicted_class": "AI" if secondary_model_prediction == 1 else "HUMAN",
-        "main_model_probability": str(main_model_probability)
+        "main_model_probability": str(main_model_probability),
+        "secondary_model_prediction": secondary_model_prediction,
+        "confidence": get_confidence(main_model_probability, secondary_model_prediction)
     }
+
+
+def get_confidence(main_model_output: float, secondary_model_output: int):
+    if (main_model_output >= 0.8 and secondary_model_output == 1) or (main_model_output <= 0.2 and secondary_model_output == 0):
+        return 'High Confidence'
+    elif (0.5 < main_model_output < 0.8 and secondary_model_output == 1) or (0.2 < main_model_output <= 0.5 and secondary_model_output == 0):
+        return 'Partially Confident'
+    else:
+        return 'Low Confidence'

core-model-prediction/random_forest_model.py

@@ -14,8 +14,8 @@ class RandomForestModel:
         ]
 
     def preprocess_input(self, secondary_model_features: List[float]) -> np.ndarray:
-        features_df = pd.DataFrame([secondary_model_features], columns=[
-                                   self.secondary_model_features])
+        features_df = pd.DataFrame(
+            [secondary_model_features], columns=self.secondary_model_features)
         features_df[self.secondary_model_features] = self.scaler.transform(
             features_df[self.secondary_model_features])
         return features_df.values.astype(np.float32).reshape(1, -1)

core-model-prediction/requirements.txt

@@ -5,9 +5,6 @@ textstat==0.7.3
 scikit-learn==1.2.2
 transformers==4.38.2
 sentence-transformers==2.7.0
-langchain 
-openai 
-langchain-openai
 fastapi
 uvicorn
 google-cloud-secret-manager

core-model-prediction/secondary_model_dependencies.py

@@ -1,35 +1,19 @@
 from sentence_transformers import SentenceTransformer, util
 from collections import Counter
-from langchain_openai import ChatOpenAI
-from langchain_core.messages import HumanMessage, SystemMessage
-from google.cloud import secretmanager
 
 
 class SecondaryModelDependencies:
     def __init__(self):
         self.text_similarity_model = SentenceTransformer(
             'sentence-transformers/all-mpnet-base-v2')
-        api_key = self.access_openai_api_key()
-        self.llm_gpt35 = ChatOpenAI(
-            api_key=api_key, model="gpt-3.5-turbo")
-        self.llm_gpt4 = ChatOpenAI(
-            api_key=api_key, model="gpt-4-turbo")
 
-    def access_openai_api_key(self):
-        client = secretmanager.SecretManagerServiceClient()
-        name = "projects/steady-climate-416810/secrets/OPENAI_API_KEY/versions/1"
-        response = client.access_secret_version(request={"name": name})
-        return response.payload.data.decode('UTF-8')
-
-    def calculate_features(self, question: str, answer: str, probability: float, backspace_count: int, typing_duration: int, letter_click_counts: dict[str, int]):
+    def calculate_features(self, question: str, answer: str, probability: float, backspace_count: int, typing_duration: int,
+                           letter_click_counts: dict[str, int], gpt35_answer: str, gpt4_answer: str):
         backspace_count_normalized = backspace_count / len(answer)
         typing_duration_normalized = typing_duration / len(answer)
         letter_discrepancy = self.calculate_letter_discrepancy(
             answer, letter_click_counts)
 
-        gpt35_answer = self.generate_gpt35_answer(question)
-        gpt4_answer = self.generate_gpt4_answer(question)
-
         cosine_sim_gpt35 = self.calculate_similarity_gpt35(
             answer, gpt35_answer)
         cosine_sim_gpt4 = self.calculate_similarity_gpt4(answer, gpt4_answer)
@@ -54,26 +38,6 @@ class SecondaryModelDependencies:
 
         return discrepancy_ratio_normalized
 
-    def generate_gpt35_answer(self, question: str):
-        messages = [
-            SystemMessage(
-                content="Please answer the following question based solely on your internal knowledge, without external references. Assume you are the human."),
-            HumanMessage(question)
-        ]
-
-        gpt35_answer = self.llm_gpt35.invoke(messages)
-        return gpt35_answer.content
-
-    def generate_gpt4_answer(self, question: str):
-        messages = [
-            SystemMessage(
-                content="Please answer the following question based solely on your internal knowledge, without external references. Assume you are the human."),
-            HumanMessage(question)
-        ]
-
-        gpt4_answer = self.llm_gpt4.invoke(messages)
-        return gpt4_answer.content
-
     def calculate_similarity_gpt35(self, answer: str, gpt35_answer: str) -> float:
         embedding1 = self.text_similarity_model.encode(
             [answer], convert_to_tensor=True)