Update vectorizer.py

controllers/vectorizer.py

@@ -1,6 +1,7 @@
 """Module to upsert data into AstraDB"""
 import os
 import logging
+import uuid
 
 import pandas as pd
 from langchain_astradb import AstraDBVectorStore
@@ -52,6 +53,10 @@ def vectorize(article):
         is_separator_regex=False,
     )
 
-    docs = text_splitter.split_documents(documents)
-    inserted_ids = vstore.add_documents(docs)
+    chunks = text_splitter.split_documents(documents)
+    ids = []
+    for chunk in chunks:
+        id = f"{chunk.metadata['id']}-{str(uuid.uuid5(uuid.NAMESPACE_OID,chunk.page_content))}"
+        ids.append(id)
+    inserted_ids = vstore.add_documents(chunks, ids=ids)
     logging.info(inserted_ids)