Update app.py
Browse files
app.py
CHANGED
@@ -6,6 +6,8 @@ from hazm import *
|
|
6 |
import docx
|
7 |
from transformers import AutoTokenizer, AutoModel
|
8 |
from langchain.llms import OpenAI
|
|
|
|
|
9 |
|
10 |
# بارگذاری مدلها و توکنایزر
|
11 |
tokenizer = AutoTokenizer.from_pretrained("HooshvareLab/bert-fa-base-uncased")
|
@@ -13,7 +15,7 @@ model = AutoModel.from_pretrained("HooshvareLab/bert-fa-base-uncased")
|
|
13 |
|
14 |
@st.cache
|
15 |
def get_embedding(text):
|
16 |
-
|
17 |
with torch.no_grad():
|
18 |
outputs = model(**inputs)
|
19 |
embeddings = outputs.last_hidden_state.mean(dim=1)
|
|
|
6 |
import docx
|
7 |
from transformers import AutoTokenizer, AutoModel
|
8 |
from langchain.llms import OpenAI
|
9 |
+
from langchain.chat_models import ChatOpenAI
|
10 |
+
|
11 |
|
12 |
# بارگذاری مدلها و توکنایزر
|
13 |
tokenizer = AutoTokenizer.from_pretrained("HooshvareLab/bert-fa-base-uncased")
|
|
|
15 |
|
16 |
@st.cache
|
17 |
def get_embedding(text):
|
18 |
+
inputs = tokenizer(text, return_tensors="pt", truncation=True, padding=True, max_length=512)
|
19 |
with torch.no_grad():
|
20 |
outputs = model(**inputs)
|
21 |
embeddings = outputs.last_hidden_state.mean(dim=1)
|