Spaces:
Runtime error
Runtime error
File size: 1,010 Bytes
65bb9f9 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 |
from datasets import load_from_disk
from qdrant_client import QdrantClient
from sentence_transformers import SentenceTransformer
from backend.database.qdrant import QdrantDatabase, ScientificPapersCollectionNames
if __name__ == '__main__':
client = QdrantClient(url="http://localhost:6333")
model = SentenceTransformer("sentence-transformers/multi-qa-mpnet-base-dot-v1", device="cuda")
qdrant_database = QdrantDatabase(client, model, upload_batch_size=16)
#qdrant_database.reset_database()
sample = 1000
#dataset_sample = load_from_disk('../../data/armac_scientific_papers/dataset')
#dataset_sample = dataset_sample.select(range(sample))
#dataset_sample = dataset_sample.map(lambda x: {"sections": [[j.replace('</S>', '') for j in i] for i in x["sections"]]})
#qdrant_database._bulk_upload(dataset_sample)
qdrant_database.client.create_snapshot(ScientificPapersCollectionNames.MAIN)
qdrant_database.client.create_snapshot(ScientificPapersCollectionNames.CHUNKS)
|