login to HF

hf_retrieval.py

@@ -7,6 +7,7 @@ from datetime import datetime
 from collections import defaultdict
 from datasets import load_dataset
 from qdrant_client import QdrantClient
+from huggingface_hub import login
 
 tokenizer = open_clip.get_tokenizer("ViT-B-32")
 
@@ -20,6 +21,7 @@ qdrant = QdrantClient(url=os.environ["QDRANT_CLOUD_URL"],
     prefer_grpc=False)
 collection_name = "video_chunks"
 
+login(token=os.environ["HF_API_TOKEN"])
 
 def timestamp_to_seconds(ts):
     h, m, s = ts.split(":")
@@ -90,7 +92,7 @@ def merge_chunks(chunks):
 
 
 def get_video_segment(video_id, start_time, end_time):
-    dataset = load_dataset("aegean-ai/ai-lectures-spring-24", split="train", streaming=True, use_auth_token=os.environ["HF_API_TOKEN"])
+    dataset = load_dataset("aegean-ai/ai-lectures-spring-24", split="train", streaming=True)
     for sample in dataset:
         if sample["__key__"] == video_id:
             break