download private data

app.py

@@ -33,10 +33,31 @@ st.markdown("""
 # ---------------------------------------
 # paths
 # ---------------------------------------
-base_path = "data/"
+HOME = "/home/user/app"
+
+data_dir = HOME +"/data"
+private_data_dir = HOME + "/private_data"  # Relative path in your Space
+
+# getting private data
+os.makedirs(private_data_dir, exist_ok=True)
+token = os.getenv("data")
+local_repo_path = snapshot_download(
+    repo_id="my-ai-university/data",
+    use_auth_token=token,
+    repo_type="dataset",
+    local_dir=private_data_dir,
+)
+
+adapter_path = HOME + "/LLaMA-TOMMI-1.0/"
+
 base_model_path = "meta-llama/Llama-3.2-11B-Vision-Instruct"
 base_model_path_3B = "meta-llama/Llama-3.2-3B-Instruct"
-adapter_path = "./LLaMA-TOMMI-1.0/"
+
+
+# base_path = "data/"
+# base_model_path = "meta-llama/Llama-3.2-11B-Vision-Instruct"
+# base_model_path_3B = "meta-llama/Llama-3.2-3B-Instruct"
+# adapter_path = "./LLaMA-TOMMI-1.0/"
 
 st.title(":red[AI University] :gray[/] FEM")
 # st.markdown("### Finite Element Method")
@@ -169,7 +190,7 @@ with col1:
 with col2:
     if st.button("Random Question"):
         while True:
-            random_question = get_random_question(base_path + "/questions.txt")
+            random_question = get_random_question(data_dir + "/questions.txt")
             if random_question != st.session_state.question:
                 break
         st.session_state.question = random_question
@@ -203,9 +224,9 @@ with st.spinner("Loading LLaMA-3.2-3B..."):
             st.session_state.llama_tokenizer_3B = llama_tokenizer_3B
 
 # Load YouTube and LaTeX data
-text_data_YT, context_embeddings_YT = load_youtube_data(base_path, embedding_model, yt_chunk_tokens, yt_overlap_tokens)
-text_data_Latex, context_embeddings_Latex = load_book_data(base_path, embedding_model, latex_chunk_tokens, latex_overlap_tokens)
-summary = load_summary('data/KG_FEM_summary.json')
+text_data_YT, context_embeddings_YT = load_youtube_data(data_dir, embedding_model, yt_chunk_tokens, yt_overlap_tokens)
+text_data_Latex, context_embeddings_Latex = load_book_data(private_data_dir, embedding_model, latex_chunk_tokens, latex_overlap_tokens)
+summary = load_summary(data_dir + '/KG_FEM_summary.json')
 
 if 'question_answered' not in st.session_state:
     st.session_state.question_answered = False