Upload 4 files

Dockerfile

@@ -0,0 +1,21 @@
+FROM python:3.11.11-bookworm
+
+# Set working directory
+WORKDIR /app
+
+# Copy only requirements first to leverage Docker layer caching
+COPY requirements.txt /app/
+
+
+# Install dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy the rest of the application code
+COPY . /app
+
+# Switch to a non-root user (optional)
+# RUN useradd -m appuser && chown -R appuser /app
+# USER appuser
+
+# Default command (optional, replace "app.py" with your app's entry point)
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

app.py

@@ -0,0 +1,205 @@
+from openai import OpenAI
+import google.generativeai as genai
+from crawler import extract_data
+import time
+import os
+from dotenv import load_dotenv
+import gradio as gr
+# from together import Together
+# from transformers import AutoModel, AutoTokenizer
+# from sklearn.metrics.pairwise import cosine_similarity
+# import torch
+# 
+# load_dotenv("../.env")
+# os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+# together_client = Together(
+#     api_key=os.getenv("TOGETHER_API_KEY"),
+# )
+
+genai.configure(api_key=os.getenv("GEMINI_API_KEY"))
+
+gemini_query = genai.GenerativeModel('gemini-2.0-flash-exp')
+
+gemini_summarizer = genai.GenerativeModel('gemini-1.5-flash')
+
+perplexity_client = OpenAI(api_key=os.getenv("PERPLEXITY_API_KEY"), base_url="https://api.perplexity.ai")
+# gpt_client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
+
+# with torch.no_grad():
+#     model = AutoModel.from_pretrained('BM-K/KoSimCSE-roberta')
+#     tokenizer = AutoTokenizer.from_pretrained('BM-K/KoSimCSE-roberta')
+
+# def cal_score(input_data):
+#     similarity_scores = []
+#     # Initialize model and tokenizer inside the function
+#     with torch.no_grad():
+#         inputs = tokenizer(input_data, padding=True, truncation=True, return_tensors="pt")
+#         outputs = model.get_input_embeddings()(inputs["input_ids"])
+        
+#         for ind in range(1, outputs.size(0)):
+#             a, b = outputs[0], outputs[ind]
+            
+#             a = a.reshape(1, -1)
+#             b = b.reshape(1, -1)
+
+#             a_norm = torch.nn.functional.normalize(a, p=2, dim=1)
+#             b_norm = torch.nn.functional.normalize(b, p=2, dim=1)
+
+#             similarity_scores.append(cosine_similarity(a_norm, b_norm)) # Scalar value
+#     return similarity_scores
+
+def get_answers( query: str ):
+    context = extract_data(query, 1)
+    # if len(context) > 1:
+    #     scores = cal_score( [query] + [answer['questionDetails'] for answer in context] )
+    #     context = [context for _, context in sorted(zip(scores, context), key=lambda x: x[0], reverse=True)]
+    #     mean_score = sum(scores) / len(scores)
+    #     context = [ctx for score, ctx in zip(scores, context) if score >= mean_score]
+    return context
+
+def get_gemini_query( message: str ):
+    print(">>> Starting gemini query generation...")
+
+    response = gemini_query.generate_content(message)
+
+    print("Finished gemini query generation: ", response.text)
+    return response.text
+
+def get_naver_answers( message: str ):
+    print(">>> Starting naver extraction...")
+    print("Question: ", message)
+
+    if len(message) > 300:
+        message = get_gemini_query(f"{message}\n 위의 내용을 짧은 제목으로 요약합니다. 제목만 보여주세요. 대답하지 마세요. 한국어로만 답변해주세요!!!")
+        
+        print( "Query: ", message)
+
+    context = get_answers( message )
+
+    sorted_answers = [
+        f"{index}. 질문: {answer['questionDetails']}" + '\n' + f" 답변: {'. '.join(answer['answers'])} " + '\n'
+        for (index, answer) in enumerate(context)
+    ]
+
+    document = '\n'.join(sorted_answers)
+    return document
+
+def get_perplexity_answer( message: str ):
+    print(">>> Starting perplexity extraction...")
+    messages = [
+        {
+            "role": "system",
+            "content": (
+                "You are an artificial intelligence assistant and you need to "
+                "engage in a helpful, CONCISE, polite question-answer conversation with a user."
+            ),
+        },
+        {   
+            "role": "user",
+            "content": (
+                message
+            ),
+        },
+    ]
+    response = perplexity_client.chat.completions.create(
+        model="llama-3.1-sonar-small-128k-online",
+        messages=messages
+    )
+    return response.choices[0].message.content
+
+
+def chatFunction( history ):
+    # MAX_TOKEN_LIMIT = 58000
+    start_time = time.time()
+    message = history[-1][0]
+    # content = f' 질문과 답변으로 구성된 문서를 드리겠습니다. \
+    #             아래에 제공된 질문에 답하기 위해 중요한 정보를 추출하세요. \
+    #             한국어로만 답변하세요. 구체적이지만 간결하게 작성하세요. \
+    #             실제 보험상담사가 답변을 하듯이 친절한 답변을 해 주세요. \n 질문: {message}\n 문서: '
+    
+    content = f' 보험설계사가 답을 줘서, 더 많은 질문이나 합당한 보험에 가입할 수 있도록 답변을 하려고 합니다. \
+                문서에 있는 제3자 언급을 1인칭으로 ​​바꾸세요. 예를 들어 "KB손해보험 설계사 OOO입니다" 등 제3자가 언급된 경우 "보험기관입니다"로 대체합니다. \
+                이러한 답변을 통해서 질문자가 이 답변을 보고 보험설계사에게 더 신뢰를 갖고 추가 질문이 있으면 물어볼 수 있도록 하려고 합니다. \
+                실제 보험상담사가 답변을 하듯이 친절한 답변을 해 주세요. \n 질문: {message}\n 문서: '
+    naver_docs = get_naver_answers( message )
+    print(len(naver_docs))
+
+    # if len(naver_docs) > MAX_TOKEN_LIMIT:
+    #     print("HERE")
+    #     start_tmp = time.time()
+    #     overlap = 200
+    #     answers = []
+    #     split_len = len(naver_docs) // ( ( len(naver_docs) - MAX_TOKEN_LIMIT ) // MAX_TOKEN_LIMIT + 2 ) + 1
+    #     print(len(naver_docs) // split_len) 
+    #     for i in range( len(naver_docs) // split_len ):
+    #         print("HERE: ", i)
+    #         if i == 0:
+    #             split = naver_docs[:split_len]
+    #         else:
+    #             split = naver_docs[i * split_len - overlap: (i + 1) * split_len]
+    #         answer, _ = get_qwen_small_answer(f"Summarize important points in a paragraph, given the information below, using only Korean language. Give me only the summary!!! \n {split}")
+    #         answers.append(answer)
+    #     print("Answers: ", answers)
+    #     naver_docs = '\n'.join(answers)
+    #     naver_time_taken += time.time() - start_tmp
+
+    # print("Post chunking length: ", len(naver_docs) )
+    content += "\n Naver 문서: " + naver_docs
+    
+
+    ### Extracting from Perplexity ###
+
+    perplexity_resp = get_perplexity_answer( message )
+    content += "\n Perplexity 문서: " + perplexity_resp
+
+    print(">>> Starting Gemini summarization...")
+
+    response = gemini_summarizer.generate_content( content, stream=True )
+
+    history[-1][1] = ''
+    ans = ""
+    for chunk in response:
+        ans += chunk.text.replace("*", "")
+        yield ans.strip() + "\n"
+        time.sleep(0.05)
+    
+    print("Finished Gemini summarization")
+    print("Time taken: ", time.time() - start_time)
+
+def set_user_response( message: str, history: list ):
+    history.append( [message, None] )
+    return '', history
+
+### Server-side code ###
+
+from fastapi import FastAPI
+from fastapi.responses import StreamingResponse
+from pydantic import BaseModel
+from fastapi.middleware.cors import CORSMiddleware
+
+app = FastAPI()
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=['*'],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+@app.get("/")
+async def root():
+    return {"message": "Hello World"}
+
+class Message(BaseModel):
+    message: str
+
+@app.post("/chat")
+async def chat( message: Message ):
+    history = [[message.message, None]]
+    return StreamingResponse(
+        chatFunction(history),
+        media_type='text/event-stream'
+    )
+

crawler.py

@@ -0,0 +1,135 @@
+from bs4 import BeautifulSoup
+import re
+import requests as r
+from html2text import html2text
+import tqdm
+import time
+
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.support.wait import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from webdriver_manager.chrome import ChromeDriverManager
+import multiprocessing
+
+
+# def from_desktop_to_mobile_version(url):
+#     """Convert a desktop URL to its mobile version."""
+#     return url.replace("https://kin.naver.com", "https://m.kin.naver.com")
+
+def initialize_webdriver():
+    """Initialize and return a WebDriver instance with headless options."""
+    options = webdriver.ChromeOptions()
+    options.add_argument("--headless=new")
+    options.add_argument("--disable-gpu")
+
+    service = Service(ChromeDriverManager().install())
+    return webdriver.Chrome(options=options, service=service)
+
+def process_url(url):
+    driver = initialize_webdriver()
+    try:
+        print("Processing URL:", url)
+        driver.get(url)
+        closeBtn = WebDriverWait(driver, 5).until(
+            EC.element_to_be_clickable((By.CSS_SELECTOR, ".layer_promotion_choice_inner > .ico_close_layer")),
+            message="Close button not found."
+        )
+        if closeBtn:
+            print("Closing the popup")
+            closeBtn.click()
+            time.sleep(0.2)
+            print("CLOSED")
+
+        expandBtn = driver.find_element(By.ID, 'nextPageButton')
+        print("Expand button: ", expandBtn)
+        if expandBtn.is_displayed():
+            WebDriverWait(driver, 10).until(
+                EC.element_to_be_clickable(expandBtn),
+                message="Expand button wasn't loaded in time."
+            )
+            expandBtn.click()
+            print("Clicked the ex`pand button")
+            time.sleep(0.5)
+            
+        html_content = driver.page_source
+        soup = BeautifulSoup(html_content, "html.parser")
+        
+        answers = soup.find_all('div', {'class': 'answerDetail'})
+        answers = [html2text(str(answer.prettify())) for answer in answers]
+
+        title = soup.find('div', {'class': 'endTitleSection'}).text.strip()
+
+        questionDetails = soup.find('div', {'class': 'questionDetail'}).text.strip()
+
+        title = title.replace("질문", '').strip()
+        
+        print("Answers extracted from: \n", url)
+        print(len(answers))
+        print('-'*60)
+
+        return {
+            "title": title,
+            "questionDetails": questionDetails,
+            "url": url,
+            "answers": answers
+        }
+    except Exception as e:
+        print(f"Error processing URL {url} \n\n\n{e}")
+        with open('error_urls.txt', 'w') as f:
+            f.write(url + '\n')
+        return {"title": '', "questionDetails": '', "url": url, "answers": ''}
+    finally:
+        driver.quit()
+
+
+def get_answers(results_a_elements, query):
+    """Fetch answers for all the extracted result links."""
+    if not results_a_elements:
+        print("No results found.")
+        return []
+
+    print("Result links extracted: ", len(results_a_elements))
+    
+    # Limit the number of parallel processes for better resource management
+    max_processes = 4
+    
+    with multiprocessing.Pool(processes=max_processes) as pool:
+        results = pool.map(process_url, results_a_elements)
+
+    # results = []
+    # # answer_count = 0
+    # for url in tqdm.tqdm(results_a_elements):
+    #     res = process_url(url)
+    #     results.append(res)
+        # answer_count += len(res['answers'])
+    return results
+
+def get_search_results(query, num_pages):
+    """Fetch search results for the given query from Naver 지식in."""
+    results = []
+    for page in range(1, num_pages + 1):
+        url = f"https://kin.naver.com/search/list.naver?query={query}&page={page}"
+        print("Starting the scraping process for:\n", url)
+
+        try:
+            response = r.get(url)
+            soup = BeautifulSoup(response.text, "html.parser")
+            results_a_elements = soup.find("ul", {"class": "basic1"}).find_all("a", {"class": "_searchListTitleAnchor"})
+            results_a_elements = [a.get('href') for a in results_a_elements if a.get("href")]
+            results += results_a_elements
+        except Exception as e:
+            print(f"Error while fetching search results: {e}")
+    return results
+
+def extract_data(query, num_pages=150) -> list[dict[str, object]]:
+    results_a_elements = get_search_results(query, num_pages)
+    print(results_a_elements)
+    answers = get_answers(results_a_elements, query)
+    print("Total answers collected:", len(answers))
+    return answers
+
+
+# if __name__ == "__main__":
+#     process_url("https://kin.naver.com/qna/detail.naver?d1id=4&dirId=401030203&docId=478845808&qb=67O07ZeYIOyImOyIoOu5hA==&enc=utf8&section=kin.qna_ency&rank=1&search_sort=0&spq=0")

requirements.txt

@@ -0,0 +1,21 @@
+beautifulsoup4
+selenium
+webdriver-manager
+fastapi[standard]
+# uvicorn[standard]
+html2text
+# transformers
+openai
+google-genai
+# transformers[torch]
+# torch
+# torchvision
+# torchaudio
+gradio
+# scikit-learn
+together
+python-dotenv
+openpyxl
+# tonic-validate
+google-generativeai
+uvicorn